Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the AWS Config source (#120834)

Tom Owersf36a94f72174
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 68e123b4de8b..470173276c7a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1142,13 +1142,13 @@ Today (7): `aggregated_resources`, `config_rules`, `conformance_pack_compliance_
Diffed against: <https://raw.githubusercontent.com/boto/botocore/develop/botocore/data/config/2014-11-12/service-2.json>
-- [ ] `Resource configuration history (GetResourceConfigHistory, POST /)` — provides configuration changes, deleted resource states, and historical relationships for auditing and incident analysis (high)
+- [x] `Resource configuration history (GetResourceConfigHistory, POST /)` — provides configuration changes, deleted resource states, and historical relationships for auditing and incident analysis (high)
- [x] `Rule evaluation results (GetComplianceDetailsByConfigRule, POST /)` — connects synced rules to evaluated resources, compliance outcomes, timestamps, and reasons for failures (high)
-- [ ] `Conformance pack evaluation results (GetConformancePackComplianceDetails, POST /)` — connects synced packs, rules, and resources to detailed compliance outcomes (high)
+- [x] `Conformance pack evaluation results (GetConformancePackComplianceDetails, POST /)` — connects synced packs, rules, and resources to detailed compliance outcomes (high)
- [x] `Conformance pack compliance scores (ListConformancePackComplianceScores, POST /)` — provides the percentage of compliant rule-resource combinations for compliance dashboards (high)
- [x] `Aggregated resource configurations (SelectAggregateResourceConfig, POST /)` — enables resource inventory and configuration analysis across accounts and regions (high)
-- [ ] `Remediation execution status (DescribeRemediationExecutionStatus, POST /)` — shows remediation progress, step timestamps, and errors for noncompliant resources (medium)
-- [ ] `Proactive resource evaluations (ListResourceEvaluations, POST /)` — tracks compliance evaluations before resources are deployed (medium)
+- [x] `Remediation execution status (DescribeRemediationExecutionStatus, POST /)` — shows remediation progress, step timestamps, and errors for noncompliant resources (medium)
+- [x] `Proactive resource evaluations (ListResourceEvaluations, POST /)` — tracks compliance evaluations before resources are deployed (medium)
Note: Compared the official botocore operation model with the static endpoint map and extraction code. Current tables contain regional snapshots and rule summaries; history and detailed evaluations require calls for each resource or rule. Aggregated resource queries require aggregator discovery and separate traversal. JSON RPC reads use POST /; recorder settings, delivery channels, and stored queries were excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/aws_config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/aws_config.py
index 8ef55e5c112c..be9763481809 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/aws_config.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/aws_config.py
@@ -39,6 +39,9 @@ _TIMESTAMP_COLUMNS = {
"result_recorded_time",
"config_rule_invoked_time",
"ordering_timestamp",
+ "configuration_item_delivery_time",
+ "invocation_time",
+ "evaluation_start_timestamp",
}
# Config reads use POST, so the transport must retry this method for transient HTTP failures.
@@ -58,6 +61,8 @@ class AwsConfigResumeConfig:
# Fan-out tables: the parent page to fetch again, and the parent that `next_token` continues.
parent_next_token: str | None = None
parent_key: str | None = None
+ # Incremental start time of the interrupted attempt, because page tokens belong to the original filter.
+ since: float | None = None
class AwsConfigError(Exception):
@@ -139,15 +144,45 @@ class AwsConfigClient:
return parsed
-def request_payload(endpoint: AwsConfigEndpoint, *, probe: bool = False) -> dict[str, Any]:
+def request_payload(endpoint: AwsConfigEndpoint, *, probe: bool = False, since: float | None = None) -> dict[str, Any]:
payload: dict[str, Any] = dict(endpoint.params or {})
if endpoint.page_size is not None:
- payload["Limit"] = 1 if probe else endpoint.page_size
+ payload[endpoint.limit_key] = 1 if probe else endpoint.page_size
if endpoint.expression is not None:
payload["Expression"] = endpoint.expression
+ if since is not None and endpoint.incremental_filter:
+ *parents, leaf = endpoint.incremental_filter
+ target = payload
+ for key in parents:
+ target = target.setdefault(key, {})
+ target[leaf] = since
return payload
+def epoch_seconds(value: Any) -> float:
+ if isinstance(value, str):
+ value = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
+ if isinstance(value, dt.datetime):
+ return (value if value.tzinfo else value.replace(tzinfo=dt.UTC)).timestamp()
+ if isinstance(value, dt.date):
+ return dt.datetime.combine(value, dt.time(), tzinfo=dt.UTC).timestamp()
+ if isinstance(value, int | float) and not isinstance(value, bool):
+ return float(value)
+ raise ValueError(f"Unsupported AWS Config incremental value: {value!r}")
+
+
+def incremental_start(
+ endpoint: AwsConfigEndpoint, should_use_incremental_field: bool, last_value: Any, now: dt.datetime
+) -> float | None:
+ if endpoint.incremental_field is None:
+ return None
+ if should_use_incremental_field and last_value is not None:
+ return epoch_seconds(last_value)
+ if endpoint.initial_window is not None:
+ return (now - endpoint.initial_window).timestamp()
+ return None
+
+
def snake_case(key: str) -> str:
return _CAMEL_BOUNDARY.sub("_", key).lower()
@@ -178,7 +213,7 @@ def parse_page(
if not isinstance(item, dict):
raise ValueError("AWS Config returned an invalid resource. Try the sync again.")
rows.append(normalize_row(item, region, endpoint.flatten))
- token = body.get("NextToken") or None
+ token = body.get(endpoint.token_key) or None
if token is not None and (not isinstance(token, str) or token == previous_token):
raise ValueError("AWS Config returned an invalid page token. Try the sync again.")
return rows, token
@@ -190,10 +225,13 @@ def request_page(
payload: dict[str, Any],
manager: "ResumableSourceManager[AwsConfigResumeConfig]",
resumed: bool,
+ empty_error_codes: frozenset[str] = frozenset(),
) -> dict[str, Any]:
try:
return client.request(operation, payload)
except AwsConfigError as error:
+ if error.code in empty_error_codes:
+ return {}
if resumed and error.code == "InvalidNextTokenException":
# A fresh retry must restart the full-refresh destination as well as extraction.
manager.clear_state()
@@ -205,17 +243,18 @@ def get_page_rows(
endpoint: AwsConfigEndpoint,
manager: "ResumableSourceManager[AwsConfigResumeConfig]",
resume: AwsConfigResumeConfig | None,
+ since: float | None,
) -> Generator[list[dict[str, Any]]]:
next_token = resume.next_token if resume else None
restarting = next_token is not None
while True:
- payload = request_payload(endpoint)
+ payload = request_payload(endpoint, since=since)
if next_token:
- payload["NextToken"] = next_token
+ payload[endpoint.token_key] = next_token
body = request_page(client, endpoint.operation, payload, manager, restarting)
restarting = False
rows, next_token = parse_page(endpoint, body, client.region, next_token)
- manager.save_state(AwsConfigResumeConfig(next_token=next_token, complete=next_token is None))
+ manager.save_state(AwsConfigResumeConfig(next_token=next_token, complete=next_token is None, since=since))
if rows:
yield rows
manager.safe_point()
@@ -223,16 +262,37 @@ def get_page_rows(
break
+def parent_values(parent: AwsConfigEndpoint, item: Any, params: tuple[str, ...]) -> tuple[str, ...]:
+ if parent.expression is not None and isinstance(item, str):
+ try:
+ item = json.loads(item)
+ except ValueError:
+ item = None
+ values: list[str] = []
+ for param in params:
+ value = item.get(param) if isinstance(item, dict) else None
+ if not isinstance(value, str) or not value:
+ raise ValueError("AWS Config returned an invalid parent resource. Try the sync again.")
+ values.append(value)
+ return tuple(values)
+
+
+def parent_key(values: tuple[str, ...]) -> str:
+ # Single-value keys stay plain strings so checkpoints saved before composite keys still resume.
+ return values[0] if len(values) == 1 else json.dumps(values)
+
+
def get_fan_out_rows(
client: AwsConfigClient,
endpoint: AwsConfigEndpoint,
manager: "ResumableSourceManager[AwsConfigResumeConfig]",
resume: AwsConfigResumeConfig | None,
+ since: float | None,
) -> Generator[list[dict[str, Any]]]:
- parent, param = endpoint.parent, endpoint.parent_param
- if parent is None or param is None:
+ parent, params = endpoint.parent, endpoint.parent_params
+ if parent is None or not params:
raise ValueError(f"AWS Config table {endpoint.operation} has no parent.")
- column = snake_case(param)
+ columns = [snake_case(param) for param in params]
parent_token = resume.parent_next_token if resume else None
resume_key = resume.parent_key if resume else None
child_token = resume.next_token if resume else None
@@ -241,13 +301,12 @@ def get_fan_out_rows(
while True:
payload = request_payload(parent)
if parent_token:
- payload["NextToken"] = parent_token
+ payload[parent.token_key] = parent_token
body = request_page(client, parent.operation, payload, manager, restarting_parent)
restarting_parent = False
- names = [item.get(param) if isinstance(item, dict) else None for item in body.get(parent.result_key) or []]
- if not all(isinstance(name, str) and name for name in names):
- raise ValueError("AWS Config returned an invalid parent resource. Try the sync again.")
- next_parent_token = body.get("NextToken") or None
+ parents = [parent_values(parent, item, params) for item in body.get(parent.result_key) or []]
+ names = [parent_key(values) for values in parents]
+ next_parent_token = body.get(parent.token_key) or None
if next_parent_token is not None and (
not isinstance(next_parent_token, str) or next_parent_token == parent_token
):
@@ -260,30 +319,35 @@ def get_fan_out_rows(
resume_key = None
if not names:
manager.save_state(
- AwsConfigResumeConfig(parent_next_token=next_parent_token)
+ AwsConfigResumeConfig(parent_next_token=next_parent_token, since=since)
if next_parent_token is not None
else AwsConfigResumeConfig(complete=True)
)
manager.safe_point()
for index in range(start, len(names)):
name = names[index]
+ values = parents[index]
if index + 1 < len(names):
- done = AwsConfigResumeConfig(parent_next_token=parent_token, parent_key=names[index + 1])
+ done = AwsConfigResumeConfig(parent_next_token=parent_token, parent_key=names[index + 1], since=since)
elif next_parent_token is not None:
- done = AwsConfigResumeConfig(parent_next_token=next_parent_token)
+ done = AwsConfigResumeConfig(parent_next_token=next_parent_token, since=since)
else:
done = AwsConfigResumeConfig(complete=True)
while True:
- child_payload = {**request_payload(endpoint), param: name}
+ child_payload = {**request_payload(endpoint, since=since), **dict(zip(params, values))}
if child_token:
- child_payload["NextToken"] = child_token
- child_body = request_page(client, endpoint.operation, child_payload, manager, restarting_child)
+ child_payload[endpoint.token_key] = child_token
+ child_body = request_page(
+ client, endpoint.operation, child_payload, manager, restarting_child, endpoint.empty_error_codes
+ )
restarting_child = False
rows, child_token = parse_page(endpoint, child_body, client.region, child_token)
for row in rows:
- row[column] = name
+ row.update(zip(columns, values))
manager.save_state(
- AwsConfigResumeConfig(parent_next_token=parent_token, parent_key=name, next_token=child_token)
+ AwsConfigResumeConfig(
+ parent_next_token=parent_token, parent_key=name, next_token=child_token, since=since
+ )
if child_token is not None
else done
)
@@ -302,17 +366,20 @@ def get_rows(
endpoint: AwsConfigEndpoint,
api_version: str,
manager: "ResumableSourceManager[AwsConfigResumeConfig]",
+ since: float | None = None,
) -> Generator[list[dict[str, Any]]]:
resume = manager.load_state() if manager.can_resume() else None
if resume is not None and resume.complete:
manager.clear_state()
return
+ if resume is not None and resume.since is not None:
+ since = resume.since
client = AwsConfigClient(config, api_version)
try:
if endpoint.parent is None:
- yield from get_page_rows(client, endpoint, manager, resume)
+ yield from get_page_rows(client, endpoint, manager, resume, since)
else:
- yield from get_fan_out_rows(client, endpoint, manager, resume)
+ yield from get_fan_out_rows(client, endpoint, manager, resume, since)
manager.clear_state()
finally:
client.close()
@@ -332,12 +399,19 @@ def validate_credentials(
operation = probe.operation
try:
body = client.request(operation, request_payload(probe, probe=True))
- if endpoint.parent is not None and endpoint.parent_param is not None:
+ if endpoint.parent is not None and endpoint.parent_params:
parents = body.get(endpoint.parent.result_key) or []
- name = parents[0].get(endpoint.parent_param) if parents and isinstance(parents[0], dict) else None
- if name:
+ if parents:
+ values = parent_values(endpoint.parent, parents[0], endpoint.parent_params)
operation = endpoint.operation
- client.request(operation, {**request_payload(endpoint, probe=True), endpoint.parent_param: name})
+ try:
+ client.request(
+ operation,
+ {**request_payload(endpoint, probe=True), **dict(zip(endpoint.parent_params, values))},
+ )
+ except AwsConfigError as error:
+ if error.code not in endpoint.empty_error_codes:
+ raise
except AwsConfigError as error:
if error.code in {"AccessDenied", "AccessDeniedException"}:
if schema_name is None:
@@ -356,13 +430,24 @@ def aws_config_source(
endpoint: str,
api_version: str,
manager: "ResumableSourceManager[AwsConfigResumeConfig]",
+ should_use_incremental_field: bool = False,
+ db_incremental_field_last_value: Any = None,
) -> SourceResponse:
endpoint_config = AWS_CONFIG_ENDPOINTS.get(endpoint)
if endpoint_config is None:
raise ValueError(f"Unknown AWS Config table: {endpoint}")
+ since = incremental_start(
+ endpoint_config, should_use_incremental_field, db_incremental_field_last_value, dt.datetime.now(dt.UTC)
+ )
+ partition_key = endpoint_config.partition_key
return SourceResponse(
name=endpoint,
- items=lambda: get_rows(config, endpoint_config, api_version, manager),
+ items=lambda: get_rows(config, endpoint_config, api_version, manager, since),
primary_keys=list(endpoint_config.primary_key),
+ partition_count=1 if partition_key else None,
+ partition_size=1 if partition_key else None,
+ partition_mode="datetime" if partition_key else None,
+ partition_format="month" if partition_key else None,
+ partition_keys=[partition_key] if partition_key else None,
sort_mode=None,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/canonical_descriptions.py
index 6b7f9846bd4a..efae5243c620 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/canonical_descriptions.py
@@ -106,4 +106,73 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"tags": "Tags attached to the resource.",
},
},
+ "resource_config_history": {
+ "description": "Configuration items that AWS Config recorded for each current resource, one row for each recorded state.",
+ "docs_url": "https://docs.aws.amazon.com/config/latest/APIReference/API_GetResourceConfigHistory.html",
+ "columns": {
+ "version": "Version number of the resource configuration.",
+ "account_id": "AWS account that owns the resource.",
+ "aws_region": "AWS region where the resource exists.",
+ "region": "AWS region selected for this import.",
+ "availability_zone": "Availability Zone of the resource.",
+ "arn": "Amazon Resource Name of the resource.",
+ "resource_id": "Resource identifier assigned by its AWS service.",
+ "resource_type": "AWS resource type, such as AWS::EC2::Instance.",
+ "resource_name": "Resource name, when the service provides one.",
+ "configuration_item_status": "Status of the configuration item, such as OK, ResourceDiscovered, or ResourceDeleted.",
+ "configuration_state_id": "Identifier that orders the configuration items of a resource.",
+ "configuration_item_md5_hash": "MD5 hash of the configuration item state.",
+ "configuration_item_capture_time": "Time when AWS Config recorded this configuration.",
+ "configuration_item_delivery_time": "Time when AWS Config delivered this configuration change, when available.",
+ "resource_creation_time": "Time when the resource was created, when available.",
+ "configuration": "Resource configuration as a JSON-encoded string.",
+ "supplementary_configuration": "Additional configuration properties for some resource types.",
+ "tags": "Tags attached to the resource.",
+ "related_events": "CloudTrail event IDs that caused this configuration change.",
+ "relationships": "Related AWS resources and how they relate to this resource.",
+ "recording_frequency": "Recording frequency, CONTINUOUS or DAILY.",
+ },
+ },
+ "conformance_pack_evaluation_results": {
+ "description": "Latest evaluation result for each rule and resource in each conformance pack.",
+ "docs_url": "https://docs.aws.amazon.com/config/latest/APIReference/API_GetConformancePackComplianceDetails.html",
+ "columns": {
+ "conformance_pack_name": "Name of the conformance pack.",
+ "config_rule_name": "Name of the conformance pack rule that evaluated the resource.",
+ "resource_type": "AWS resource type of the evaluated resource.",
+ "resource_id": "Identifier of the evaluated resource.",
+ "evaluation_mode": "Evaluation mode, DETECTIVE or PROACTIVE.",
+ "ordering_timestamp": "Time of the event that triggered the evaluation.",
+ "resource_evaluation_id": "Unique identifier of the evaluation result.",
+ "compliance_type": "Evaluation outcome, COMPLIANT or NON_COMPLIANT.",
+ "config_rule_invoked_time": "Time when the rule ran the evaluation.",
+ "result_recorded_time": "Time when AWS Config recorded the evaluation result.",
+ "annotation": "Additional information about how the evaluation determined compliance.",
+ "region": "AWS region selected for this import.",
+ },
+ },
+ "remediation_execution_statuses": {
+ "description": "State and step details of the latest remediation execution for each resource and AWS Config rule.",
+ "docs_url": "https://docs.aws.amazon.com/config/latest/APIReference/API_DescribeRemediationExecutionStatus.html",
+ "columns": {
+ "config_rule_name": "Name of the AWS Config rule that has the remediation.",
+ "resource_type": "AWS resource type of the remediated resource.",
+ "resource_id": "Identifier of the remediated resource.",
+ "state": "State of the remediation execution, such as QUEUED, IN_PROGRESS, SUCCEEDED, or FAILED.",
+ "step_details": "Name, state, start time, stop time, and error message of each remediation step.",
+ "invocation_time": "Time when the remediation execution started.",
+ "last_updated_time": "Time when the remediation execution was last updated.",
+ "region": "AWS region selected for this import.",
+ },
+ },
+ "resource_evaluations": {
+ "description": "Proactive evaluations of resource configurations, usually run before the resources are deployed.",
+ "docs_url": "https://docs.aws.amazon.com/config/latest/APIReference/API_ListResourceEvaluations.html",
+ "columns": {
+ "resource_evaluation_id": "Identifier of the resource evaluation.",
+ "evaluation_mode": "Evaluation mode, PROACTIVE for these evaluations.",
+ "evaluation_start_timestamp": "Time when the evaluation started.",
+ "region": "AWS region selected for this import.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/settings.py
index d850ad85aa69..54468da87c2c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/settings.py
@@ -1,8 +1,12 @@
+import datetime as dt
from collections.abc import Mapping
from typing import Any
from posthog.dataclasses import frozen
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
+from products.warehouse_sources.backend.types import IncrementalField
+
CONFIG_API_VERSION = "2014-11-12"
TARGET_PREFIXES = {CONFIG_API_VERSION: "StarlingDoveService"}
@@ -22,9 +26,19 @@ class AwsConfigEndpoint:
expression: str | None = None
params: Mapping[str, Any] | None = None
flatten: tuple[str, ...] = ()
- # Child endpoints run once per parent row, passing the parent's `parent_param` value through.
+ limit_key: str = "Limit"
+ token_key: str = "NextToken"
+ # Child endpoints run once per parent row, passing the parent's `parent_params` values through.
parent: "AwsConfigEndpoint | None" = None
- parent_param: str | None = None
+ parent_params: tuple[str, ...] = ()
+ # Error codes that mean one parent has nothing to read, such as a rule without remediation.
+ empty_error_codes: frozenset[str] = frozenset()
+ incremental_field: str | None = None
+ # Request path that takes the incremental start time, in epoch seconds.
+ incremental_filter: tuple[str, ...] = ()
+ # Start time for a read without an incremental value. None reads everything the API keeps.
+ initial_window: dt.timedelta | None = None
+ partition_key: str | None = None
CONFIG_RULES = AwsConfigEndpoint(
@@ -34,6 +48,23 @@ CONFIG_RULES = AwsConfigEndpoint(
description="AWS Config rules, their evaluation settings, and resource scopes.",
)
+CONFORMANCE_PACKS = AwsConfigEndpoint(
+ operation="DescribeConformancePacks",
+ result_key="ConformancePackDetails",
+ primary_key=("conformance_pack_arn",),
+ description="Conformance packs and their deployment settings in the selected region.",
+ page_size=20,
+)
+
+RESOURCE_KEYS = AwsConfigEndpoint(
+ operation="SelectResourceConfig",
+ result_key="Results",
+ primary_key=("resource_type", "resource_id"),
+ description="Type and identifier of each recorded resource.",
+ page_size=100,
+ expression="SELECT resourceType, resourceId",
+)
+
CONFIGURATION_AGGREGATORS = AwsConfigEndpoint(
operation="DescribeConfigurationAggregators",
result_key="ConfigurationAggregators",
@@ -59,13 +90,7 @@ AWS_CONFIG_ENDPOINTS: dict[str, AwsConfigEndpoint] = {
primary_key=("region", "config_rule_name"),
description="Current compliance status and counts of noncompliant resources for each rule.",
),
- "conformance_packs": AwsConfigEndpoint(
- operation="DescribeConformancePacks",
- result_key="ConformancePackDetails",
- primary_key=("conformance_pack_arn",),
- description="Conformance packs and their deployment settings in the selected region.",
- page_size=20,
- ),
+ "conformance_packs": CONFORMANCE_PACKS,
"conformance_pack_compliance_scores": AwsConfigEndpoint(
operation="ListConformancePackComplianceScores",
result_key="ConformancePackComplianceScores",
@@ -83,7 +108,7 @@ AWS_CONFIG_ENDPOINTS: dict[str, AwsConfigEndpoint] = {
params={"ComplianceTypes": ["COMPLIANT", "NON_COMPLIANT", "NOT_APPLICABLE"]},
flatten=("EvaluationResultIdentifier", "EvaluationResultQualifier"),
parent=CONFIG_RULES,
- parent_param="ConfigRuleName",
+ parent_params=("ConfigRuleName",),
),
"aggregated_resources": AwsConfigEndpoint(
operation="SelectAggregateResourceConfig",
@@ -93,12 +118,71 @@ AWS_CONFIG_ENDPOINTS: dict[str, AwsConfigEndpoint] = {
page_size=100,
expression=RESOURCE_EXPRESSION,
parent=CONFIGURATION_AGGREGATORS,
- parent_param="ConfigurationAggregatorName",
+ parent_params=("ConfigurationAggregatorName",),
+ ),
+ "resource_config_history": AwsConfigEndpoint(
+ operation="GetResourceConfigHistory",
+ result_key="configurationItems",
+ primary_key=("account_id", "aws_region", "resource_type", "resource_id", "configuration_state_id"),
+ description=(
+ "Configuration items recorded for each current resource in the selected region. "
+ "Without a saved incremental value, the table reads the last 90 days."
+ ),
+ page_size=100,
+ limit_key="limit",
+ token_key="nextToken",
+ parent=RESOURCE_KEYS,
+ parent_params=("resourceType", "resourceId"),
+ empty_error_codes=frozenset({"ResourceNotDiscoveredException"}),
+ incremental_field="configuration_item_capture_time",
+ incremental_filter=("earlierTime",),
+ # One history call covers at most seven days, so reading years of history for each resource is too slow.
+ initial_window=dt.timedelta(days=90),
+ partition_key="configuration_item_capture_time",
+ ),
+ "conformance_pack_evaluation_results": AwsConfigEndpoint(
+ operation="GetConformancePackComplianceDetails",
+ result_key="ConformancePackRuleEvaluationResults",
+ primary_key=("region", "conformance_pack_name", "config_rule_name", "resource_type", "resource_id"),
+ description="Latest evaluation result for each rule and resource in each conformance pack.",
+ page_size=100,
+ flatten=("EvaluationResultIdentifier", "EvaluationResultQualifier"),
+ parent=CONFORMANCE_PACKS,
+ parent_params=("ConformancePackName",),
+ empty_error_codes=frozenset({"NoSuchConformancePackException"}),
+ ),
+ "remediation_execution_statuses": AwsConfigEndpoint(
+ operation="DescribeRemediationExecutionStatus",
+ result_key="RemediationExecutionStatuses",
+ primary_key=("region", "config_rule_name", "resource_type", "resource_id"),
+ description="State and step details of the latest remediation execution for each resource and rule.",
+ page_size=100,
+ flatten=("ResourceKey",),
+ parent=CONFIG_RULES,
+ parent_params=("ConfigRuleName",),
+ empty_error_codes=frozenset({"NoSuchRemediationConfigurationException"}),
+ ),
+ "resource_evaluations": AwsConfigEndpoint(
+ operation="ListResourceEvaluations",
+ result_key="ResourceEvaluations",
+ primary_key=("resource_evaluation_id",),
+ description="Proactive evaluations of resource configurations, usually run before deployment.",
+ page_size=100,
+ incremental_field="evaluation_start_timestamp",
+ incremental_filter=("Filters", "TimeWindow", "StartTime"),
+ partition_key="evaluation_start_timestamp",
),
}
ENDPOINTS = tuple(AWS_CONFIG_ENDPOINTS)
ENDPOINT_DESCRIPTIONS = {name: endpoint.description for name, endpoint in AWS_CONFIG_ENDPOINTS.items()}
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ name: [incremental_field(endpoint.incremental_field)]
+ for name, endpoint in AWS_CONFIG_ENDPOINTS.items()
+ if endpoint.incremental_field is not None
+}
+# A configuration item can reach the API after its capture time, so each sync reads the last hour again.
+INCREMENTAL_LOOKBACK_SECONDS = {"resource_config_history": 3600}
ERROR_MESSAGES = {
"AccessDenied": "AWS denied access. Grant the config read permission for the selected table to this IAM user or role.",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/source.py
index 3b0b97475cf2..7cf17eea6e40 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/source.py
@@ -21,6 +21,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_config
ENDPOINT_DESCRIPTIONS,
ENDPOINTS,
ERROR_MESSAGES,
+ INCREMENTAL_FIELDS,
+ INCREMENTAL_LOOKBACK_SECONDS,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
@@ -73,7 +75,15 @@ class AwsConfigSource(ResumableSource[AwsConfigSourceConfig, AwsConfigResumeConf
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, {}, names, descriptions=ENDPOINT_DESCRIPTIONS)
+ return build_endpoint_schemas(
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ names,
+ # The lookback reads rows again, so appending would duplicate them.
+ merge_only=INCREMENTAL_LOOKBACK_SECONDS.keys(),
+ descriptions=ENDPOINT_DESCRIPTIONS,
+ default_incremental_lookback_seconds=INCREMENTAL_LOOKBACK_SECONDS,
+ )
def validate_credentials(
self,
@@ -94,7 +104,12 @@ class AwsConfigSource(ResumableSource[AwsConfigSourceConfig, AwsConfigResumeConf
inputs: SourceInputs,
) -> SourceResponse:
return aws_config_source(
- config, inputs.schema_name, self.resolve_api_version(inputs.api_version), resumable_source_manager
+ config,
+ inputs.schema_name,
+ self.resolve_api_version(inputs.api_version),
+ resumable_source_manager,
+ should_use_incremental_field=inputs.should_use_incremental_field,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,
)
@property
@@ -103,7 +118,7 @@ class AwsConfigSource(ResumableSource[AwsConfigSourceConfig, AwsConfigResumeConf
name=ExternalDataSourceType.AWSCONFIG,
category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,
label="AWS Config",
- caption="""Sync resource configurations, rules, compliance status and evaluation results, conformance packs, and aggregated resources from one AWS region.
+ caption="""Sync resource configurations and their history, rules, compliance status and evaluation results, conformance packs, remediation executions, proactive evaluations, and aggregated resources from one AWS region.
Grant these IAM permissions for the tables you want to sync:
- `config:SelectResourceConfig`
@@ -113,9 +128,14 @@ Grant these IAM permissions for the tables you want to sync:
- `config:DescribeConformancePacks`
- `config:ListConformancePackComplianceScores`
- `config:DescribeConfigurationAggregators` and `config:SelectAggregateResourceConfig`
+- `config:SelectResourceConfig` and `config:GetResourceConfigHistory`
+- `config:DescribeConformancePacks` and `config:GetConformancePackComplianceDetails`
+- `config:DescribeConfigRules` and `config:DescribeRemediationExecutionStatus`
+- `config:ListResourceEvaluations`
Enable an AWS Config recorder for resource inventory. The resources table contains current configurations and excludes deleted resources.
The aggregated resources table reads every configuration aggregator in the selected region.
+The resource configuration history table reads the history of each current resource. A full refresh or a first incremental sync reads the last 90 days.
Add a session token if you use temporary credentials. Replace temporary credentials when they expire.""",
iconPath="/static/services/aws_config.png",
docsUrl="https://docs.aws.amazon.com/config/latest/APIReference/Welcome.html",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/tests/test_aws_config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/tests/test_aws_config.py
index 2663606fec57..345e3e6a4b40 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/tests/test_aws_config.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/tests/test_aws_config.py
@@ -4,6 +4,7 @@ from collections.abc import Iterable, Iterator
from typing import Any, cast
import pytest
+import time_machine
from unittest.mock import MagicMock, patch
import requests
@@ -245,6 +246,38 @@ FAN_OUT_CASES = [
}
),
),
+ (
+ "conformance_pack_evaluation_results",
+ ("DescribeConformancePacks", "ConformancePackDetails", "ConformancePackName", 20),
+ ("GetConformancePackComplianceDetails", "ConformancePackRuleEvaluationResults"),
+ {"Limit": 100},
+ lambda parent, index: {
+ "ComplianceType": "COMPLIANT",
+ "EvaluationResultIdentifier": {
+ "EvaluationResultQualifier": {
+ "ConfigRuleName": f"{parent}-rule",
+ "ResourceType": "AWS::S3::Bucket",
+ "ResourceId": f"bucket-{index}",
+ },
+ "OrderingTimestamp": 1735689600,
+ },
+ "ConfigRuleInvokedTime": 1735689600,
+ "ResultRecordedTime": 1735689600,
+ },
+ ),
+ (
+ "remediation_execution_statuses",
+ ("DescribeConfigRules", "ConfigRules", "ConfigRuleName", None),
+ ("DescribeRemediationExecutionStatus", "RemediationExecutionStatuses"),
+ {"Limit": 100},
+ lambda parent, index: {
+ "ResourceKey": {"resourceType": "AWS::S3::Bucket", "resourceId": f"bucket-{index}"},
+ "State": "FAILED",
+ "StepDetails": [{"Name": "step-1", "State": "FAILED", "ErrorMessage": "Example failure"}],
+ "InvocationTime": 1735689600,
+ "LastUpdatedTime": 1735689600,
+ },
+ ),
]
@@ -273,9 +306,12 @@ def test_fan_out_walks_every_parent_and_child_page(
("c", "bucket-3"),
]
assert all(row.get(key) is not None for row in rows for key in response.primary_keys or [])
- if endpoint == "rule_evaluation_results":
+ if endpoint in ("rule_evaluation_results", "conformance_pack_evaluation_results"):
assert rows[0]["ordering_timestamp"] == dt.datetime(2025, 1, 1, tzinfo=dt.UTC)
assert "evaluation_result_identifier" not in rows[0]
+ if endpoint == "remediation_execution_statuses":
+ assert rows[0]["invocation_time"] == dt.datetime(2025, 1, 1, tzinfo=dt.UTC)
+ assert "resource_key" not in rows[0]
parent_params = {} if parent_limit is None else {"Limit": parent_limit}
expected_requests = [
(parent_operation, parent_params),
@@ -315,6 +351,149 @@ def test_fan_out_resumes_at_saved_parent(session: MagicMock) -> None:
manager.clear_state.assert_called_once()
+def history_item(resource_id: str, state: str) -> dict[str, Any]:
+ return {
+ "accountId": "111111111111",
+ "awsRegion": "eu-west-1",
+ "resourceType": "AWS::S3::Bucket",
+ "resourceId": resource_id,
+ "configurationStateId": state,
+ "configurationItemStatus": "OK",
+ "configurationItemCaptureTime": 1735689600,
+ "configuration": '{"versioning":"Enabled"}',
+ }
+
+
+def test_resource_history_walks_composite_parent_keys_and_skips_undiscovered(session: MagicMock) -> None:
+ resource = '{{"resourceType":"AWS::{0}","resourceId":"{1}"}}'
+ session.post.side_effect = [
+ make_response(
+ {
+ "Results": [resource.format("S3::Bucket", "bucket-1"), resource.format("S3::Bucket", "bucket-2")],
+ "NextToken": "parent-2",
+ }
+ ),
+ make_response({"configurationItems": [history_item("bucket-1", "3")], "nextToken": "history-2"}),
+ make_response({"configurationItems": [history_item("bucket-1", "2")]}),
+ make_response({"__type": "ResourceNotDiscoveredException", "message": "Unknown resource"}, 400),
+ make_response({"Results": [resource.format("S3::Bucket", "bucket-3")]}),
+ make_response({"configurationItems": [history_item("bucket-3", "1")]}),
+ ]
+ manager = make_manager()
+ response = aws_config_source(
+ make_config(), "resource_config_history", "2014-11-12", manager, True, dt.datetime(2025, 1, 1, tzinfo=dt.UTC)
+ )
+ rows = [row for batch in cast(Iterable[Any], response.items()) for row in batch]
+
+ assert [(row["resource_id"], row["configuration_state_id"]) for row in rows] == [
+ ("bucket-1", "3"),
+ ("bucket-1", "2"),
+ ("bucket-3", "1"),
+ ]
+ assert rows[0]["configuration_item_capture_time"] == dt.datetime(2025, 1, 1, tzinfo=dt.UTC)
+ assert all(row.get(key) is not None for row in rows for key in response.primary_keys or [])
+ assert response.partition_keys == ["configuration_item_capture_time"]
+ child = {"limit": 100, "earlierTime": 1735689600.0, "resourceType": "AWS::S3::Bucket"}
+ assert [
+ (call.kwargs["headers"]["X-Amz-Target"].split(".")[1], json.loads(call.kwargs["data"]))
+ for call in session.post.call_args_list
+ ] == [
+ ("SelectResourceConfig", {"Limit": 100, "Expression": "SELECT resourceType, resourceId"}),
+ ("GetResourceConfigHistory", {**child, "resourceId": "bucket-1"}),
+ ("GetResourceConfigHistory", {**child, "resourceId": "bucket-1", "nextToken": "history-2"}),
+ ("GetResourceConfigHistory", {**child, "resourceId": "bucket-2"}),
+ (
+ "SelectResourceConfig",
+ {"Limit": 100, "Expression": "SELECT resourceType, resourceId", "NextToken": "parent-2"},
+ ),
+ ("GetResourceConfigHistory", {**child, "resourceId": "bucket-3"}),
+ ]
+ assert [call.args[0] for call in manager.save_state.call_args_list] == [
+ AwsConfigResumeConfig(parent_key='["AWS::S3::Bucket", "bucket-1"]', next_token="history-2", since=1735689600.0),
+ AwsConfigResumeConfig(parent_key='["AWS::S3::Bucket", "bucket-2"]', since=1735689600.0),
+ AwsConfigResumeConfig(parent_next_token="parent-2", since=1735689600.0),
+ AwsConfigResumeConfig(complete=True),
+ ]
+
+
+@time_machine.travel(dt.datetime(2025, 4, 1, tzinfo=dt.UTC), tick=False)
+ "endpoint,should_use_incremental_field,last_value,expected_filter",
+ [
+ (
+ "resource_evaluations",
+ True,
+ dt.datetime(2025, 1, 1, tzinfo=dt.UTC),
+ {"Filters": {"TimeWindow": {"StartTime": 1735689600.0}}},
+ ),
+ (
+ "resource_evaluations",
+ True,
+ "2025-01-01T00:00:00Z",
+ {"Filters": {"TimeWindow": {"StartTime": 1735689600.0}}},
+ ),
+ ("resource_evaluations", True, None, {}),
+ ("resource_evaluations", False, dt.datetime(2025, 1, 1, tzinfo=dt.UTC), {}),
+ ("resource_config_history", True, dt.datetime(2025, 1, 1), {"earlierTime": 1735689600.0}),
+ ("resource_config_history", True, dt.date(2025, 1, 1), {"earlierTime": 1735689600.0}),
+ ("resource_config_history", False, dt.datetime(2025, 3, 1, tzinfo=dt.UTC), {"earlierTime": 1735689600.0}),
+ ("resource_config_history", True, None, {"earlierTime": 1735689600.0}),
+ ("config_rules", True, dt.datetime(2025, 1, 1, tzinfo=dt.UTC), {}),
+ ],
+)
+def test_incremental_start_shapes_request(
+ session: MagicMock,
+ endpoint: str,
+ should_use_incremental_field: bool,
+ last_value: Any,
+ expected_filter: dict[str, Any],
+) -> None:
+ parent_page = make_response({"Results": ['{"resourceType":"AWS::S3::Bucket","resourceId":"bucket-1"}']})
+ pages = [make_response({})]
+ session.post.side_effect = [parent_page, *pages] if endpoint == "resource_config_history" else pages
+ manager = make_manager()
+ response = aws_config_source(
+ make_config(), endpoint, "2014-11-12", manager, should_use_incremental_field, last_value
+ )
+ list(cast(Iterable[Any], response.items()))
+
+ payload = json.loads(session.post.call_args_list[-1].kwargs["data"])
+ for key in ("Limit", "limit", "resourceType", "resourceId", "Expression"):
+ payload.pop(key, None)
+ assert payload == expected_filter
+
+
+def test_resume_keeps_incremental_start_of_interrupted_attempt(session: MagicMock) -> None:
+ session.post.side_effect = [
+ make_response(
+ {
+ "Results": [
+ '{"resourceType":"AWS::S3::Bucket","resourceId":"bucket-1"}',
+ '{"resourceType":"AWS::S3::Bucket","resourceId":"bucket-2"}',
+ ]
+ }
+ ),
+ make_response({"configurationItems": []}),
+ ]
+ manager = make_manager(
+ AwsConfigResumeConfig(parent_key='["AWS::S3::Bucket", "bucket-2"]', next_token="history-2", since=1735689600.0)
+ )
+ response = aws_config_source(
+ make_config(), "resource_config_history", "2014-11-12", manager, True, dt.datetime(2025, 2, 1, tzinfo=dt.UTC)
+ )
+ list(cast(Iterable[Any], response.items()))
+
+ payloads = [json.loads(call.kwargs["data"]) for call in session.post.call_args_list]
+ assert payloads[1] == {
+ "limit": 100,
+ "earlierTime": 1735689600.0,
+ "resourceType": "AWS::S3::Bucket",
+ "resourceId": "bucket-2",
+ "nextToken": "history-2",
+ }
+ manager.clear_state.assert_called_once()
+
+
def test_fan_out_missing_saved_parent_clears_state_and_fails_attempt(session: MagicMock) -> None:
session.post.return_value = make_response({"ConfigRules": [{"ConfigRuleName": "a"}]})
manager = make_manager(AwsConfigResumeConfig(parent_key="deleted", next_token="child-2"))
@@ -470,18 +649,34 @@ def test_credential_errors_are_actionable(code: str, message: str) -> None:
[{"ConfigurationAggregators": [{"ConfigurationAggregatorName": "example-aggregator"}]}, {}],
[("DescribeConfigurationAggregators", 1), ("SelectAggregateResourceConfig", 1)],
),
+ (
+ "resource_config_history",
+ [{"Results": ['{"resourceType":"AWS::S3::Bucket","resourceId":"example-bucket"}']}, {}],
+ [("SelectResourceConfig", 1), ("GetResourceConfigHistory", 1)],
+ ),
+ (
+ "remediation_execution_statuses",
+ [
+ {"ConfigRules": [{"ConfigRuleName": "example-rule"}]},
+ ScriptedResponse(status=400, json={"__type": "NoSuchRemediationConfigurationException"}),
+ ],
+ [("DescribeConfigRules", None), ("DescribeRemediationExecutionStatus", 1)],
+ ),
+ ("resource_evaluations", [{}], [("ListResourceEvaluations", 1)]),
],
)
def test_credential_probe_reads_parent_then_one_child(
- schema: str | None, responses: list[dict[str, Any]], expected: list[tuple[str, int | None]]
+ schema: str | None, responses: list[Any], expected: list[tuple[str, int | None]]
) -> None:
- with scripted_network([ScriptedResponse(json=body) for body in responses]) as network:
+ script = [body if isinstance(body, ScriptedResponse) else ScriptedResponse(json=body) for body in responses]
+ with scripted_network(script) as network:
assert validate_credentials(make_config(), "2014-11-12", schema) == (True, None)
assert [
- (request.headers["x-amz-target"].split(".")[1], request.json().get("Limit")) for request in network.requests_log
+ (request.headers["x-amz-target"].split(".")[1], request.json().get("Limit", request.json().get("limit")))
+ for request in network.requests_log
] == expected
if len(expected) == 2:
- assert set(network.requests_log[1].json()) & {"ConfigRuleName", "ConfigurationAggregatorName"}
+ assert set(network.requests_log[1].json()) & {"ConfigRuleName", "ConfigurationAggregatorName", "resourceId"}
def test_fan_out_access_denied_names_child_permission() -> None: