:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the AWS Config source (#120834)
Tom Owersf36a94f72174
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 68e123b4de8b..470173276c7a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -1142,13 +1142,13 @@ Today (7): `aggregated_resources`, `config_rules`, `conformance_pack_compliance_Diffed against: <https://raw.githubusercontent.com/boto/botocore/develop/botocore/data/config/2014-11-12/service-2.json>-- [ ] `Resource configuration history (GetResourceConfigHistory, POST /)` — provides configuration changes, deleted resource states, and historical relationships for auditing and incident analysis (high)+- [x] `Resource configuration history (GetResourceConfigHistory, POST /)` — provides configuration changes, deleted resource states, and historical relationships for auditing and incident analysis (high)- [x] `Rule evaluation results (GetComplianceDetailsByConfigRule, POST /)` — connects synced rules to evaluated resources, compliance outcomes, timestamps, and reasons for failures (high)-- [ ] `Conformance pack evaluation results (GetConformancePackComplianceDetails, POST /)` — connects synced packs, rules, and resources to detailed compliance outcomes (high)+- [x] `Conformance pack evaluation results (GetConformancePackComplianceDetails, POST /)` — connects synced packs, rules, and resources to detailed compliance outcomes (high)- [x] `Conformance pack compliance scores (ListConformancePackComplianceScores, POST /)` — provides the percentage of compliant rule-resource combinations for compliance dashboards (high)- [x] `Aggregated resource configurations (SelectAggregateResourceConfig, POST /)` — enables resource inventory and configuration analysis across accounts and regions (high)-- [ ] `Remediation execution status (DescribeRemediationExecutionStatus, POST /)` — shows remediation progress, step timestamps, and errors for noncompliant resources (medium)-- [ ] `Proactive resource evaluations (ListResourceEvaluations, POST /)` — tracks compliance evaluations before resources are deployed (medium)+- [x] `Remediation execution status (DescribeRemediationExecutionStatus, POST /)` — shows remediation progress, step timestamps, and errors for noncompliant resources (medium)+- [x] `Proactive resource evaluations (ListResourceEvaluations, POST /)` — tracks compliance evaluations before resources are deployed (medium)Note: Compared the official botocore operation model with the static endpoint map and extraction code. Current tables contain regional snapshots and rule summaries; history and detailed evaluations require calls for each resource or rule. Aggregated resource queries require aggregator discovery and separate traversal. JSON RPC reads use POST /; recorder settings, delivery channels, and stored queries were excluded.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/aws_config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/aws_config.pyindex 8ef55e5c112c..be9763481809 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/aws_config.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/aws_config.py@@ -39,6 +39,9 @@ _TIMESTAMP_COLUMNS = {"result_recorded_time","config_rule_invoked_time","ordering_timestamp",+ "configuration_item_delivery_time",+ "invocation_time",+ "evaluation_start_timestamp",}# Config reads use POST, so the transport must retry this method for transient HTTP failures.@@ -58,6 +61,8 @@ class AwsConfigResumeConfig:# Fan-out tables: the parent page to fetch again, and the parent that `next_token` continues.parent_next_token: str | None = Noneparent_key: str | None = None+ # Incremental start time of the interrupted attempt, because page tokens belong to the original filter.+ since: float | None = Noneclass AwsConfigError(Exception):@@ -139,15 +144,45 @@ class AwsConfigClient:return parsed-def request_payload(endpoint: AwsConfigEndpoint, *, probe: bool = False) -> dict[str, Any]:+def request_payload(endpoint: AwsConfigEndpoint, *, probe: bool = False, since: float | None = None) -> dict[str, Any]:payload: dict[str, Any] = dict(endpoint.params or {})if endpoint.page_size is not None:- payload["Limit"] = 1 if probe else endpoint.page_size+ payload[endpoint.limit_key] = 1 if probe else endpoint.page_sizeif endpoint.expression is not None:payload["Expression"] = endpoint.expression+ if since is not None and endpoint.incremental_filter:+ *parents, leaf = endpoint.incremental_filter+ target = payload+ for key in parents:+ target = target.setdefault(key, {})+ target[leaf] = sincereturn payload+def epoch_seconds(value: Any) -> float:+ if isinstance(value, str):+ value = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))+ if isinstance(value, dt.datetime):+ return (value if value.tzinfo else value.replace(tzinfo=dt.UTC)).timestamp()+ if isinstance(value, dt.date):+ return dt.datetime.combine(value, dt.time(), tzinfo=dt.UTC).timestamp()+ if isinstance(value, int | float) and not isinstance(value, bool):+ return float(value)+ raise ValueError(f"Unsupported AWS Config incremental value: {value!r}")+++def incremental_start(+ endpoint: AwsConfigEndpoint, should_use_incremental_field: bool, last_value: Any, now: dt.datetime+) -> float | None:+ if endpoint.incremental_field is None:+ return None+ if should_use_incremental_field and last_value is not None:+ return epoch_seconds(last_value)+ if endpoint.initial_window is not None:+ return (now - endpoint.initial_window).timestamp()+ return None++def snake_case(key: str) -> str:return _CAMEL_BOUNDARY.sub("_", key).lower()@@ -178,7 +213,7 @@ def parse_page(if not isinstance(item, dict):raise ValueError("AWS Config returned an invalid resource. Try the sync again.")rows.append(normalize_row(item, region, endpoint.flatten))- token = body.get("NextToken") or None+ token = body.get(endpoint.token_key) or Noneif token is not None and (not isinstance(token, str) or token == previous_token):raise ValueError("AWS Config returned an invalid page token. Try the sync again.")return rows, token@@ -190,10 +225,13 @@ def request_page(payload: dict[str, Any],manager: "ResumableSourceManager[AwsConfigResumeConfig]",resumed: bool,+ empty_error_codes: frozenset[str] = frozenset(),) -> dict[str, Any]:try:return client.request(operation, payload)except AwsConfigError as error:+ if error.code in empty_error_codes:+ return {}if resumed and error.code == "InvalidNextTokenException":# A fresh retry must restart the full-refresh destination as well as extraction.manager.clear_state()@@ -205,17 +243,18 @@ def get_page_rows(endpoint: AwsConfigEndpoint,manager: "ResumableSourceManager[AwsConfigResumeConfig]",resume: AwsConfigResumeConfig | None,+ since: float | None,) -> Generator[list[dict[str, Any]]]:next_token = resume.next_token if resume else Nonerestarting = next_token is not Nonewhile True:- payload = request_payload(endpoint)+ payload = request_payload(endpoint, since=since)if next_token:- payload["NextToken"] = next_token+ payload[endpoint.token_key] = next_tokenbody = request_page(client, endpoint.operation, payload, manager, restarting)restarting = Falserows, next_token = parse_page(endpoint, body, client.region, next_token)- manager.save_state(AwsConfigResumeConfig(next_token=next_token, complete=next_token is None))+ manager.save_state(AwsConfigResumeConfig(next_token=next_token, complete=next_token is None, since=since))if rows:yield rowsmanager.safe_point()@@ -223,16 +262,37 @@ def get_page_rows(break+def parent_values(parent: AwsConfigEndpoint, item: Any, params: tuple[str, ...]) -> tuple[str, ...]:+ if parent.expression is not None and isinstance(item, str):+ try:+ item = json.loads(item)+ except ValueError:+ item = None+ values: list[str] = []+ for param in params:+ value = item.get(param) if isinstance(item, dict) else None+ if not isinstance(value, str) or not value:+ raise ValueError("AWS Config returned an invalid parent resource. Try the sync again.")+ values.append(value)+ return tuple(values)+++def parent_key(values: tuple[str, ...]) -> str:+ # Single-value keys stay plain strings so checkpoints saved before composite keys still resume.+ return values[0] if len(values) == 1 else json.dumps(values)++def get_fan_out_rows(client: AwsConfigClient,endpoint: AwsConfigEndpoint,manager: "ResumableSourceManager[AwsConfigResumeConfig]",resume: AwsConfigResumeConfig | None,+ since: float | None,) -> Generator[list[dict[str, Any]]]:- parent, param = endpoint.parent, endpoint.parent_param- if parent is None or param is None:+ parent, params = endpoint.parent, endpoint.parent_params+ if parent is None or not params:raise ValueError(f"AWS Config table {endpoint.operation} has no parent.")- column = snake_case(param)+ columns = [snake_case(param) for param in params]parent_token = resume.parent_next_token if resume else Noneresume_key = resume.parent_key if resume else Nonechild_token = resume.next_token if resume else None@@ -241,13 +301,12 @@ def get_fan_out_rows(while True:payload = request_payload(parent)if parent_token:- payload["NextToken"] = parent_token+ payload[parent.token_key] = parent_tokenbody = request_page(client, parent.operation, payload, manager, restarting_parent)restarting_parent = False- names = [item.get(param) if isinstance(item, dict) else None for item in body.get(parent.result_key) or []]- if not all(isinstance(name, str) and name for name in names):- raise ValueError("AWS Config returned an invalid parent resource. Try the sync again.")- next_parent_token = body.get("NextToken") or None+ parents = [parent_values(parent, item, params) for item in body.get(parent.result_key) or []]+ names = [parent_key(values) for values in parents]+ next_parent_token = body.get(parent.token_key) or Noneif next_parent_token is not None and (not isinstance(next_parent_token, str) or next_parent_token == parent_token):@@ -260,30 +319,35 @@ def get_fan_out_rows(resume_key = Noneif not names:manager.save_state(- AwsConfigResumeConfig(parent_next_token=next_parent_token)+ AwsConfigResumeConfig(parent_next_token=next_parent_token, since=since)if next_parent_token is not Noneelse AwsConfigResumeConfig(complete=True))manager.safe_point()for index in range(start, len(names)):name = names[index]+ values = parents[index]if index + 1 < len(names):- done = AwsConfigResumeConfig(parent_next_token=parent_token, parent_key=names[index + 1])+ done = AwsConfigResumeConfig(parent_next_token=parent_token, parent_key=names[index + 1], since=since)elif next_parent_token is not None:- done = AwsConfigResumeConfig(parent_next_token=next_parent_token)+ done = AwsConfigResumeConfig(parent_next_token=next_parent_token, since=since)else:done = AwsConfigResumeConfig(complete=True)while True:- child_payload = {**request_payload(endpoint), param: name}+ child_payload = {**request_payload(endpoint, since=since), **dict(zip(params, values))}if child_token:- child_payload["NextToken"] = child_token- child_body = request_page(client, endpoint.operation, child_payload, manager, restarting_child)+ child_payload[endpoint.token_key] = child_token+ child_body = request_page(+ client, endpoint.operation, child_payload, manager, restarting_child, endpoint.empty_error_codes+ )restarting_child = Falserows, child_token = parse_page(endpoint, child_body, client.region, child_token)for row in rows:- row[column] = name+ row.update(zip(columns, values))manager.save_state(- AwsConfigResumeConfig(parent_next_token=parent_token, parent_key=name, next_token=child_token)+ AwsConfigResumeConfig(+ parent_next_token=parent_token, parent_key=name, next_token=child_token, since=since+ )if child_token is not Noneelse done)@@ -302,17 +366,20 @@ def get_rows(endpoint: AwsConfigEndpoint,api_version: str,manager: "ResumableSourceManager[AwsConfigResumeConfig]",+ since: float | None = None,) -> Generator[list[dict[str, Any]]]:resume = manager.load_state() if manager.can_resume() else Noneif resume is not None and resume.complete:manager.clear_state()return+ if resume is not None and resume.since is not None:+ since = resume.sinceclient = AwsConfigClient(config, api_version)try:if endpoint.parent is None:- yield from get_page_rows(client, endpoint, manager, resume)+ yield from get_page_rows(client, endpoint, manager, resume, since)else:- yield from get_fan_out_rows(client, endpoint, manager, resume)+ yield from get_fan_out_rows(client, endpoint, manager, resume, since)manager.clear_state()finally:client.close()@@ -332,12 +399,19 @@ def validate_credentials(operation = probe.operationtry:body = client.request(operation, request_payload(probe, probe=True))- if endpoint.parent is not None and endpoint.parent_param is not None:+ if endpoint.parent is not None and endpoint.parent_params:parents = body.get(endpoint.parent.result_key) or []- name = parents[0].get(endpoint.parent_param) if parents and isinstance(parents[0], dict) else None- if name:+ if parents:+ values = parent_values(endpoint.parent, parents[0], endpoint.parent_params)operation = endpoint.operation- client.request(operation, {**request_payload(endpoint, probe=True), endpoint.parent_param: name})+ try:+ client.request(+ operation,+ {**request_payload(endpoint, probe=True), **dict(zip(endpoint.parent_params, values))},+ )+ except AwsConfigError as error:+ if error.code not in endpoint.empty_error_codes:+ raiseexcept AwsConfigError as error:if error.code in {"AccessDenied", "AccessDeniedException"}:if schema_name is None:@@ -356,13 +430,24 @@ def aws_config_source(endpoint: str,api_version: str,manager: "ResumableSourceManager[AwsConfigResumeConfig]",+ should_use_incremental_field: bool = False,+ db_incremental_field_last_value: Any = None,) -> SourceResponse:endpoint_config = AWS_CONFIG_ENDPOINTS.get(endpoint)if endpoint_config is None:raise ValueError(f"Unknown AWS Config table: {endpoint}")+ since = incremental_start(+ endpoint_config, should_use_incremental_field, db_incremental_field_last_value, dt.datetime.now(dt.UTC)+ )+ partition_key = endpoint_config.partition_keyreturn SourceResponse(name=endpoint,- items=lambda: get_rows(config, endpoint_config, api_version, manager),+ items=lambda: get_rows(config, endpoint_config, api_version, manager, since),primary_keys=list(endpoint_config.primary_key),+ partition_count=1 if partition_key else None,+ partition_size=1 if partition_key else None,+ partition_mode="datetime" if partition_key else None,+ partition_format="month" if partition_key else None,+ partition_keys=[partition_key] if partition_key else None,sort_mode=None,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/canonical_descriptions.pyindex 6b7f9846bd4a..efae5243c620 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/canonical_descriptions.py@@ -106,4 +106,73 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"tags": "Tags attached to the resource.",},},+ "resource_config_history": {+ "description": "Configuration items that AWS Config recorded for each current resource, one row for each recorded state.",+ "docs_url": "https://docs.aws.amazon.com/config/latest/APIReference/API_GetResourceConfigHistory.html",+ "columns": {+ "version": "Version number of the resource configuration.",+ "account_id": "AWS account that owns the resource.",+ "aws_region": "AWS region where the resource exists.",+ "region": "AWS region selected for this import.",+ "availability_zone": "Availability Zone of the resource.",+ "arn": "Amazon Resource Name of the resource.",+ "resource_id": "Resource identifier assigned by its AWS service.",+ "resource_type": "AWS resource type, such as AWS::EC2::Instance.",+ "resource_name": "Resource name, when the service provides one.",+ "configuration_item_status": "Status of the configuration item, such as OK, ResourceDiscovered, or ResourceDeleted.",+ "configuration_state_id": "Identifier that orders the configuration items of a resource.",+ "configuration_item_md5_hash": "MD5 hash of the configuration item state.",+ "configuration_item_capture_time": "Time when AWS Config recorded this configuration.",+ "configuration_item_delivery_time": "Time when AWS Config delivered this configuration change, when available.",+ "resource_creation_time": "Time when the resource was created, when available.",+ "configuration": "Resource configuration as a JSON-encoded string.",+ "supplementary_configuration": "Additional configuration properties for some resource types.",+ "tags": "Tags attached to the resource.",+ "related_events": "CloudTrail event IDs that caused this configuration change.",+ "relationships": "Related AWS resources and how they relate to this resource.",+ "recording_frequency": "Recording frequency, CONTINUOUS or DAILY.",+ },+ },+ "conformance_pack_evaluation_results": {+ "description": "Latest evaluation result for each rule and resource in each conformance pack.",+ "docs_url": "https://docs.aws.amazon.com/config/latest/APIReference/API_GetConformancePackComplianceDetails.html",+ "columns": {+ "conformance_pack_name": "Name of the conformance pack.",+ "config_rule_name": "Name of the conformance pack rule that evaluated the resource.",+ "resource_type": "AWS resource type of the evaluated resource.",+ "resource_id": "Identifier of the evaluated resource.",+ "evaluation_mode": "Evaluation mode, DETECTIVE or PROACTIVE.",+ "ordering_timestamp": "Time of the event that triggered the evaluation.",+ "resource_evaluation_id": "Unique identifier of the evaluation result.",+ "compliance_type": "Evaluation outcome, COMPLIANT or NON_COMPLIANT.",+ "config_rule_invoked_time": "Time when the rule ran the evaluation.",+ "result_recorded_time": "Time when AWS Config recorded the evaluation result.",+ "annotation": "Additional information about how the evaluation determined compliance.",+ "region": "AWS region selected for this import.",+ },+ },+ "remediation_execution_statuses": {+ "description": "State and step details of the latest remediation execution for each resource and AWS Config rule.",+ "docs_url": "https://docs.aws.amazon.com/config/latest/APIReference/API_DescribeRemediationExecutionStatus.html",+ "columns": {+ "config_rule_name": "Name of the AWS Config rule that has the remediation.",+ "resource_type": "AWS resource type of the remediated resource.",+ "resource_id": "Identifier of the remediated resource.",+ "state": "State of the remediation execution, such as QUEUED, IN_PROGRESS, SUCCEEDED, or FAILED.",+ "step_details": "Name, state, start time, stop time, and error message of each remediation step.",+ "invocation_time": "Time when the remediation execution started.",+ "last_updated_time": "Time when the remediation execution was last updated.",+ "region": "AWS region selected for this import.",+ },+ },+ "resource_evaluations": {+ "description": "Proactive evaluations of resource configurations, usually run before the resources are deployed.",+ "docs_url": "https://docs.aws.amazon.com/config/latest/APIReference/API_ListResourceEvaluations.html",+ "columns": {+ "resource_evaluation_id": "Identifier of the resource evaluation.",+ "evaluation_mode": "Evaluation mode, PROACTIVE for these evaluations.",+ "evaluation_start_timestamp": "Time when the evaluation started.",+ "region": "AWS region selected for this import.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/settings.pyindex d850ad85aa69..54468da87c2c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/settings.py@@ -1,8 +1,12 @@+import datetime as dtfrom collections.abc import Mappingfrom typing import Anyfrom posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field+from products.warehouse_sources.backend.types import IncrementalField+CONFIG_API_VERSION = "2014-11-12"TARGET_PREFIXES = {CONFIG_API_VERSION: "StarlingDoveService"}@@ -22,9 +26,19 @@ class AwsConfigEndpoint:expression: str | None = Noneparams: Mapping[str, Any] | None = Noneflatten: tuple[str, ...] = ()- # Child endpoints run once per parent row, passing the parent's `parent_param` value through.+ limit_key: str = "Limit"+ token_key: str = "NextToken"+ # Child endpoints run once per parent row, passing the parent's `parent_params` values through.parent: "AwsConfigEndpoint | None" = None- parent_param: str | None = None+ parent_params: tuple[str, ...] = ()+ # Error codes that mean one parent has nothing to read, such as a rule without remediation.+ empty_error_codes: frozenset[str] = frozenset()+ incremental_field: str | None = None+ # Request path that takes the incremental start time, in epoch seconds.+ incremental_filter: tuple[str, ...] = ()+ # Start time for a read without an incremental value. None reads everything the API keeps.+ initial_window: dt.timedelta | None = None+ partition_key: str | None = NoneCONFIG_RULES = AwsConfigEndpoint(@@ -34,6 +48,23 @@ CONFIG_RULES = AwsConfigEndpoint(description="AWS Config rules, their evaluation settings, and resource scopes.",)+CONFORMANCE_PACKS = AwsConfigEndpoint(+ operation="DescribeConformancePacks",+ result_key="ConformancePackDetails",+ primary_key=("conformance_pack_arn",),+ description="Conformance packs and their deployment settings in the selected region.",+ page_size=20,+)++RESOURCE_KEYS = AwsConfigEndpoint(+ operation="SelectResourceConfig",+ result_key="Results",+ primary_key=("resource_type", "resource_id"),+ description="Type and identifier of each recorded resource.",+ page_size=100,+ expression="SELECT resourceType, resourceId",+)+CONFIGURATION_AGGREGATORS = AwsConfigEndpoint(operation="DescribeConfigurationAggregators",result_key="ConfigurationAggregators",@@ -59,13 +90,7 @@ AWS_CONFIG_ENDPOINTS: dict[str, AwsConfigEndpoint] = {primary_key=("region", "config_rule_name"),description="Current compliance status and counts of noncompliant resources for each rule.",),- "conformance_packs": AwsConfigEndpoint(- operation="DescribeConformancePacks",- result_key="ConformancePackDetails",- primary_key=("conformance_pack_arn",),- description="Conformance packs and their deployment settings in the selected region.",- page_size=20,- ),+ "conformance_packs": CONFORMANCE_PACKS,"conformance_pack_compliance_scores": AwsConfigEndpoint(operation="ListConformancePackComplianceScores",result_key="ConformancePackComplianceScores",@@ -83,7 +108,7 @@ AWS_CONFIG_ENDPOINTS: dict[str, AwsConfigEndpoint] = {params={"ComplianceTypes": ["COMPLIANT", "NON_COMPLIANT", "NOT_APPLICABLE"]},flatten=("EvaluationResultIdentifier", "EvaluationResultQualifier"),parent=CONFIG_RULES,- parent_param="ConfigRuleName",+ parent_params=("ConfigRuleName",),),"aggregated_resources": AwsConfigEndpoint(operation="SelectAggregateResourceConfig",@@ -93,12 +118,71 @@ AWS_CONFIG_ENDPOINTS: dict[str, AwsConfigEndpoint] = {page_size=100,expression=RESOURCE_EXPRESSION,parent=CONFIGURATION_AGGREGATORS,- parent_param="ConfigurationAggregatorName",+ parent_params=("ConfigurationAggregatorName",),+ ),+ "resource_config_history": AwsConfigEndpoint(+ operation="GetResourceConfigHistory",+ result_key="configurationItems",+ primary_key=("account_id", "aws_region", "resource_type", "resource_id", "configuration_state_id"),+ description=(+ "Configuration items recorded for each current resource in the selected region. "+ "Without a saved incremental value, the table reads the last 90 days."+ ),+ page_size=100,+ limit_key="limit",+ token_key="nextToken",+ parent=RESOURCE_KEYS,+ parent_params=("resourceType", "resourceId"),+ empty_error_codes=frozenset({"ResourceNotDiscoveredException"}),+ incremental_field="configuration_item_capture_time",+ incremental_filter=("earlierTime",),+ # One history call covers at most seven days, so reading years of history for each resource is too slow.+ initial_window=dt.timedelta(days=90),+ partition_key="configuration_item_capture_time",+ ),+ "conformance_pack_evaluation_results": AwsConfigEndpoint(+ operation="GetConformancePackComplianceDetails",+ result_key="ConformancePackRuleEvaluationResults",+ primary_key=("region", "conformance_pack_name", "config_rule_name", "resource_type", "resource_id"),+ description="Latest evaluation result for each rule and resource in each conformance pack.",+ page_size=100,+ flatten=("EvaluationResultIdentifier", "EvaluationResultQualifier"),+ parent=CONFORMANCE_PACKS,+ parent_params=("ConformancePackName",),+ empty_error_codes=frozenset({"NoSuchConformancePackException"}),+ ),+ "remediation_execution_statuses": AwsConfigEndpoint(+ operation="DescribeRemediationExecutionStatus",+ result_key="RemediationExecutionStatuses",+ primary_key=("region", "config_rule_name", "resource_type", "resource_id"),+ description="State and step details of the latest remediation execution for each resource and rule.",+ page_size=100,+ flatten=("ResourceKey",),+ parent=CONFIG_RULES,+ parent_params=("ConfigRuleName",),+ empty_error_codes=frozenset({"NoSuchRemediationConfigurationException"}),+ ),+ "resource_evaluations": AwsConfigEndpoint(+ operation="ListResourceEvaluations",+ result_key="ResourceEvaluations",+ primary_key=("resource_evaluation_id",),+ description="Proactive evaluations of resource configurations, usually run before deployment.",+ page_size=100,+ incremental_field="evaluation_start_timestamp",+ incremental_filter=("Filters", "TimeWindow", "StartTime"),+ partition_key="evaluation_start_timestamp",),}ENDPOINTS = tuple(AWS_CONFIG_ENDPOINTS)ENDPOINT_DESCRIPTIONS = {name: endpoint.description for name, endpoint in AWS_CONFIG_ENDPOINTS.items()}+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {+ name: [incremental_field(endpoint.incremental_field)]+ for name, endpoint in AWS_CONFIG_ENDPOINTS.items()+ if endpoint.incremental_field is not None+}+# A configuration item can reach the API after its capture time, so each sync reads the last hour again.+INCREMENTAL_LOOKBACK_SECONDS = {"resource_config_history": 3600}ERROR_MESSAGES = {"AccessDenied": "AWS denied access. Grant the config read permission for the selected table to this IAM user or role.",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/source.pyindex 3b0b97475cf2..7cf17eea6e40 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/source.py@@ -21,6 +21,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_configENDPOINT_DESCRIPTIONS,ENDPOINTS,ERROR_MESSAGES,+ INCREMENTAL_FIELDS,+ INCREMENTAL_LOOKBACK_SECONDS,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (@@ -73,7 +75,15 @@ class AwsConfigSource(ResumableSource[AwsConfigSourceConfig, AwsConfigResumeConfforce_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- return build_endpoint_schemas(ENDPOINTS, {}, names, descriptions=ENDPOINT_DESCRIPTIONS)+ return build_endpoint_schemas(+ ENDPOINTS,+ INCREMENTAL_FIELDS,+ names,+ # The lookback reads rows again, so appending would duplicate them.+ merge_only=INCREMENTAL_LOOKBACK_SECONDS.keys(),+ descriptions=ENDPOINT_DESCRIPTIONS,+ default_incremental_lookback_seconds=INCREMENTAL_LOOKBACK_SECONDS,+ )def validate_credentials(self,@@ -94,7 +104,12 @@ class AwsConfigSource(ResumableSource[AwsConfigSourceConfig, AwsConfigResumeConfinputs: SourceInputs,) -> SourceResponse:return aws_config_source(- config, inputs.schema_name, self.resolve_api_version(inputs.api_version), resumable_source_manager+ config,+ inputs.schema_name,+ self.resolve_api_version(inputs.api_version),+ resumable_source_manager,+ should_use_incremental_field=inputs.should_use_incremental_field,+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,)@property@@ -103,7 +118,7 @@ class AwsConfigSource(ResumableSource[AwsConfigSourceConfig, AwsConfigResumeConfname=ExternalDataSourceType.AWSCONFIG,category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,label="AWS Config",- caption="""Sync resource configurations, rules, compliance status and evaluation results, conformance packs, and aggregated resources from one AWS region.+ caption="""Sync resource configurations and their history, rules, compliance status and evaluation results, conformance packs, remediation executions, proactive evaluations, and aggregated resources from one AWS region.Grant these IAM permissions for the tables you want to sync:- `config:SelectResourceConfig`@@ -113,9 +128,14 @@ Grant these IAM permissions for the tables you want to sync:- `config:DescribeConformancePacks`- `config:ListConformancePackComplianceScores`- `config:DescribeConfigurationAggregators` and `config:SelectAggregateResourceConfig`+- `config:SelectResourceConfig` and `config:GetResourceConfigHistory`+- `config:DescribeConformancePacks` and `config:GetConformancePackComplianceDetails`+- `config:DescribeConfigRules` and `config:DescribeRemediationExecutionStatus`+- `config:ListResourceEvaluations`Enable an AWS Config recorder for resource inventory. The resources table contains current configurations and excludes deleted resources.The aggregated resources table reads every configuration aggregator in the selected region.+The resource configuration history table reads the history of each current resource. A full refresh or a first incremental sync reads the last 90 days.Add a session token if you use temporary credentials. Replace temporary credentials when they expire.""",iconPath="/static/services/aws_config.png",docsUrl="https://docs.aws.amazon.com/config/latest/APIReference/Welcome.html",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/tests/test_aws_config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/tests/test_aws_config.pyindex 2663606fec57..345e3e6a4b40 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/tests/test_aws_config.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_config/tests/test_aws_config.py@@ -4,6 +4,7 @@ from collections.abc import Iterable, Iteratorfrom typing import Any, castimport pytest+import time_machinefrom unittest.mock import MagicMock, patchimport requests@@ -245,6 +246,38 @@ FAN_OUT_CASES = [}),),+ (+ "conformance_pack_evaluation_results",+ ("DescribeConformancePacks", "ConformancePackDetails", "ConformancePackName", 20),+ ("GetConformancePackComplianceDetails", "ConformancePackRuleEvaluationResults"),+ {"Limit": 100},+ lambda parent, index: {+ "ComplianceType": "COMPLIANT",+ "EvaluationResultIdentifier": {+ "EvaluationResultQualifier": {+ "ConfigRuleName": f"{parent}-rule",+ "ResourceType": "AWS::S3::Bucket",+ "ResourceId": f"bucket-{index}",+ },+ "OrderingTimestamp": 1735689600,+ },+ "ConfigRuleInvokedTime": 1735689600,+ "ResultRecordedTime": 1735689600,+ },+ ),+ (+ "remediation_execution_statuses",+ ("DescribeConfigRules", "ConfigRules", "ConfigRuleName", None),+ ("DescribeRemediationExecutionStatus", "RemediationExecutionStatuses"),+ {"Limit": 100},+ lambda parent, index: {+ "ResourceKey": {"resourceType": "AWS::S3::Bucket", "resourceId": f"bucket-{index}"},+ "State": "FAILED",+ "StepDetails": [{"Name": "step-1", "State": "FAILED", "ErrorMessage": "Example failure"}],+ "InvocationTime": 1735689600,+ "LastUpdatedTime": 1735689600,+ },+ ),]@@ -273,9 +306,12 @@ def test_fan_out_walks_every_parent_and_child_page(("c", "bucket-3"),]assert all(row.get(key) is not None for row in rows for key in response.primary_keys or [])- if endpoint == "rule_evaluation_results":+ if endpoint in ("rule_evaluation_results", "conformance_pack_evaluation_results"):assert rows[0]["ordering_timestamp"] == dt.datetime(2025, 1, 1, tzinfo=dt.UTC)assert "evaluation_result_identifier" not in rows[0]+ if endpoint == "remediation_execution_statuses":+ assert rows[0]["invocation_time"] == dt.datetime(2025, 1, 1, tzinfo=dt.UTC)+ assert "resource_key" not in rows[0]parent_params = {} if parent_limit is None else {"Limit": parent_limit}expected_requests = [(parent_operation, parent_params),@@ -315,6 +351,149 @@ def test_fan_out_resumes_at_saved_parent(session: MagicMock) -> None:manager.clear_state.assert_called_once()+def history_item(resource_id: str, state: str) -> dict[str, Any]:+ return {+ "accountId": "111111111111",+ "awsRegion": "eu-west-1",+ "resourceType": "AWS::S3::Bucket",+ "resourceId": resource_id,+ "configurationStateId": state,+ "configurationItemStatus": "OK",+ "configurationItemCaptureTime": 1735689600,+ "configuration": '{"versioning":"Enabled"}',+ }+++def test_resource_history_walks_composite_parent_keys_and_skips_undiscovered(session: MagicMock) -> None:+ resource = '{{"resourceType":"AWS::{0}","resourceId":"{1}"}}'+ session.post.side_effect = [+ make_response(+ {+ "Results": [resource.format("S3::Bucket", "bucket-1"), resource.format("S3::Bucket", "bucket-2")],+ "NextToken": "parent-2",+ }+ ),+ make_response({"configurationItems": [history_item("bucket-1", "3")], "nextToken": "history-2"}),+ make_response({"configurationItems": [history_item("bucket-1", "2")]}),+ make_response({"__type": "ResourceNotDiscoveredException", "message": "Unknown resource"}, 400),+ make_response({"Results": [resource.format("S3::Bucket", "bucket-3")]}),+ make_response({"configurationItems": [history_item("bucket-3", "1")]}),+ ]+ manager = make_manager()+ response = aws_config_source(+ make_config(), "resource_config_history", "2014-11-12", manager, True, dt.datetime(2025, 1, 1, tzinfo=dt.UTC)+ )+ rows = [row for batch in cast(Iterable[Any], response.items()) for row in batch]++ assert [(row["resource_id"], row["configuration_state_id"]) for row in rows] == [+ ("bucket-1", "3"),+ ("bucket-1", "2"),+ ("bucket-3", "1"),+ ]+ assert rows[0]["configuration_item_capture_time"] == dt.datetime(2025, 1, 1, tzinfo=dt.UTC)+ assert all(row.get(key) is not None for row in rows for key in response.primary_keys or [])+ assert response.partition_keys == ["configuration_item_capture_time"]+ child = {"limit": 100, "earlierTime": 1735689600.0, "resourceType": "AWS::S3::Bucket"}+ assert [+ (call.kwargs["headers"]["X-Amz-Target"].split(".")[1], json.loads(call.kwargs["data"]))+ for call in session.post.call_args_list+ ] == [+ ("SelectResourceConfig", {"Limit": 100, "Expression": "SELECT resourceType, resourceId"}),+ ("GetResourceConfigHistory", {**child, "resourceId": "bucket-1"}),+ ("GetResourceConfigHistory", {**child, "resourceId": "bucket-1", "nextToken": "history-2"}),+ ("GetResourceConfigHistory", {**child, "resourceId": "bucket-2"}),+ (+ "SelectResourceConfig",+ {"Limit": 100, "Expression": "SELECT resourceType, resourceId", "NextToken": "parent-2"},+ ),+ ("GetResourceConfigHistory", {**child, "resourceId": "bucket-3"}),+ ]+ assert [call.args[0] for call in manager.save_state.call_args_list] == [+ AwsConfigResumeConfig(parent_key='["AWS::S3::Bucket", "bucket-1"]', next_token="history-2", since=1735689600.0),+ AwsConfigResumeConfig(parent_key='["AWS::S3::Bucket", "bucket-2"]', since=1735689600.0),+ AwsConfigResumeConfig(parent_next_token="parent-2", since=1735689600.0),+ AwsConfigResumeConfig(complete=True),+ ]+++@time_machine.travel(dt.datetime(2025, 4, 1, tzinfo=dt.UTC), tick=False)+ "endpoint,should_use_incremental_field,last_value,expected_filter",+ [+ (+ "resource_evaluations",+ True,+ dt.datetime(2025, 1, 1, tzinfo=dt.UTC),+ {"Filters": {"TimeWindow": {"StartTime": 1735689600.0}}},+ ),+ (+ "resource_evaluations",+ True,+ "2025-01-01T00:00:00Z",+ {"Filters": {"TimeWindow": {"StartTime": 1735689600.0}}},+ ),+ ("resource_evaluations", True, None, {}),+ ("resource_evaluations", False, dt.datetime(2025, 1, 1, tzinfo=dt.UTC), {}),+ ("resource_config_history", True, dt.datetime(2025, 1, 1), {"earlierTime": 1735689600.0}),+ ("resource_config_history", True, dt.date(2025, 1, 1), {"earlierTime": 1735689600.0}),+ ("resource_config_history", False, dt.datetime(2025, 3, 1, tzinfo=dt.UTC), {"earlierTime": 1735689600.0}),+ ("resource_config_history", True, None, {"earlierTime": 1735689600.0}),+ ("config_rules", True, dt.datetime(2025, 1, 1, tzinfo=dt.UTC), {}),+ ],+)+def test_incremental_start_shapes_request(+ session: MagicMock,+ endpoint: str,+ should_use_incremental_field: bool,+ last_value: Any,+ expected_filter: dict[str, Any],+) -> None:+ parent_page = make_response({"Results": ['{"resourceType":"AWS::S3::Bucket","resourceId":"bucket-1"}']})+ pages = [make_response({})]+ session.post.side_effect = [parent_page, *pages] if endpoint == "resource_config_history" else pages+ manager = make_manager()+ response = aws_config_source(+ make_config(), endpoint, "2014-11-12", manager, should_use_incremental_field, last_value+ )+ list(cast(Iterable[Any], response.items()))++ payload = json.loads(session.post.call_args_list[-1].kwargs["data"])+ for key in ("Limit", "limit", "resourceType", "resourceId", "Expression"):+ payload.pop(key, None)+ assert payload == expected_filter+++def test_resume_keeps_incremental_start_of_interrupted_attempt(session: MagicMock) -> None:+ session.post.side_effect = [+ make_response(+ {+ "Results": [+ '{"resourceType":"AWS::S3::Bucket","resourceId":"bucket-1"}',+ '{"resourceType":"AWS::S3::Bucket","resourceId":"bucket-2"}',+ ]+ }+ ),+ make_response({"configurationItems": []}),+ ]+ manager = make_manager(+ AwsConfigResumeConfig(parent_key='["AWS::S3::Bucket", "bucket-2"]', next_token="history-2", since=1735689600.0)+ )+ response = aws_config_source(+ make_config(), "resource_config_history", "2014-11-12", manager, True, dt.datetime(2025, 2, 1, tzinfo=dt.UTC)+ )+ list(cast(Iterable[Any], response.items()))++ payloads = [json.loads(call.kwargs["data"]) for call in session.post.call_args_list]+ assert payloads[1] == {+ "limit": 100,+ "earlierTime": 1735689600.0,+ "resourceType": "AWS::S3::Bucket",+ "resourceId": "bucket-2",+ "nextToken": "history-2",+ }+ manager.clear_state.assert_called_once()++def test_fan_out_missing_saved_parent_clears_state_and_fails_attempt(session: MagicMock) -> None:session.post.return_value = make_response({"ConfigRules": [{"ConfigRuleName": "a"}]})manager = make_manager(AwsConfigResumeConfig(parent_key="deleted", next_token="child-2"))@@ -470,18 +649,34 @@ def test_credential_errors_are_actionable(code: str, message: str) -> None:[{"ConfigurationAggregators": [{"ConfigurationAggregatorName": "example-aggregator"}]}, {}],[("DescribeConfigurationAggregators", 1), ("SelectAggregateResourceConfig", 1)],),+ (+ "resource_config_history",+ [{"Results": ['{"resourceType":"AWS::S3::Bucket","resourceId":"example-bucket"}']}, {}],+ [("SelectResourceConfig", 1), ("GetResourceConfigHistory", 1)],+ ),+ (+ "remediation_execution_statuses",+ [+ {"ConfigRules": [{"ConfigRuleName": "example-rule"}]},+ ScriptedResponse(status=400, json={"__type": "NoSuchRemediationConfigurationException"}),+ ],+ [("DescribeConfigRules", None), ("DescribeRemediationExecutionStatus", 1)],+ ),+ ("resource_evaluations", [{}], [("ListResourceEvaluations", 1)]),],)def test_credential_probe_reads_parent_then_one_child(- schema: str | None, responses: list[dict[str, Any]], expected: list[tuple[str, int | None]]+ schema: str | None, responses: list[Any], expected: list[tuple[str, int | None]]) -> None:- with scripted_network([ScriptedResponse(json=body) for body in responses]) as network:+ script = [body if isinstance(body, ScriptedResponse) else ScriptedResponse(json=body) for body in responses]+ with scripted_network(script) as network:assert validate_credentials(make_config(), "2014-11-12", schema) == (True, None)assert [- (request.headers["x-amz-target"].split(".")[1], request.json().get("Limit")) for request in network.requests_log+ (request.headers["x-amz-target"].split(".")[1], request.json().get("Limit", request.json().get("limit")))+ for request in network.requests_log] == expectedif len(expected) == 2:- assert set(network.requests_log[1].json()) & {"ConfigRuleName", "ConfigurationAggregatorName"}+ assert set(network.requests_log[1].json()) & {"ConfigRuleName", "ConfigurationAggregatorName", "resourceId"}def test_fan_out_access_denied_names_child_permission() -> None: