Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add five aws systems manager tables (#118515)

Tom Owers779553416618
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 5a517bb81b07..adac34f8bbea 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1330,16 +1330,16 @@ Note: The official SDK model was compared with ListStateMachines, ListExecutions
## AWS Systems Manager — **thin**
-Today (5): `associations`, `inventory`, `managed_instances`, `patch_baselines`, `resource_compliance_summaries`
+Today (10): `association_execution_targets`, `association_executions`, `associations`, `compliance_items`, `documents`, `inventory`, `inventory_entries`, `managed_instances`, `patch_baselines`, `resource_compliance_summaries`
Diffed against: <https://raw.githubusercontent.com/boto/botocore/develop/botocore/data/ssm/2014-11-06/service-2.json>
-- [ ] `Documents (ListDocuments, POST /)` — resolve association document names to document types, owners, versions, and supported platforms (high)
-- [ ] `Inventory entries (ListInventoryEntries, POST /)` — analyze installed applications, services, and other inventory beyond the imported AWS:InstanceInformation records (high)
-- [ ] `Compliance items (ListComplianceItems, POST /)` — identify individual noncompliant patches or associations behind resource compliance totals (high)
+- [x] `Documents (ListDocuments, POST /)` — resolve association document names to document types, owners, versions, and supported platforms (high)
+- [x] `Inventory entries (ListInventoryEntries, POST /)` — analyze installed applications, services, and other inventory beyond the imported AWS:InstanceInformation records (high)
+- [x] `Compliance items (ListComplianceItems, POST /)` — identify individual noncompliant patches or associations behind resource compliance totals (high)
- [ ] `Instance patch states (DescribeInstancePatchStates, POST /)` — measure missing patches, failed patches, and patch scan times for each managed node (high)
- [ ] `Instance patches (DescribeInstancePatches, POST /)` — analyze installed and missing patches by classification, severity, installation date, and node (high)
-- [ ] `Association executions and targets (DescribeAssociationExecutions and DescribeAssociationExecutionTargets, POST /)` — measure association execution success and identify failed target nodes over time (high)
+- [x] `Association executions and targets (DescribeAssociationExecutions and DescribeAssociationExecutionTargets, POST /)` — measure association execution success and identify failed target nodes over time (high)
- [ ] `Commands and command invocations (ListCommands and ListCommandInvocations, POST /)` — analyze remote command duration, failures, and results for each target node (medium)
- [ ] `Automation executions and steps (DescribeAutomationExecutions and DescribeAutomationStepExecutions, POST /)` — measure automation outcomes and locate slow or failed runbook steps (medium)
- [ ] `OpsItems and events (DescribeOpsItems and ListOpsItemEvents, POST /)` — track operational issues, priority, resolution times, and changes throughout their lifecycle (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/aws_systems_manager.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/aws_systems_manager.py
index 1cda1012d730..85627db2d12e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/aws_systems_manager.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/aws_systems_manager.py
@@ -1,7 +1,9 @@
import re
import json
+import hashlib
import datetime as dt
-from collections.abc import Iterator
+import dataclasses
+from collections.abc import Iterator, Sequence
from typing import Any
import requests
@@ -18,6 +20,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_system
API_VERSION,
ENDPOINTS,
ERROR_MESSAGES,
+ EXCLUDED_INVENTORY_TYPES,
+ MISSING_PARENT_ERRORS,
SystemsManagerEndpoint,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
@@ -37,11 +41,31 @@ TRANSPORT_RETRY = BoundedRetry(
)
_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")
+_PLACEHOLDER_INSTANCE_ID = "i-00000000000000000"
+_PLACEHOLDER_UUID = "00000000-0000-0000-0000-000000000000"
+# Well-formed identifiers that exist nowhere, so a probe reaches the IAM check and then fails as not found.
+PLACEHOLDER_PARENTS: dict[str, dict[str, Any]] = {
+ "resource_compliance_summaries": {
+ "ResourceId": _PLACEHOLDER_INSTANCE_ID,
+ "ResourceType": "ManagedInstance",
+ "ComplianceType": "Patch",
+ },
+ "inventory": {"Id": _PLACEHOLDER_INSTANCE_ID},
+ "associations": {"AssociationId": _PLACEHOLDER_UUID},
+ "association_executions": {"AssociationId": _PLACEHOLDER_UUID, "ExecutionId": _PLACEHOLDER_UUID},
+}
+
@frozen
class SystemsManagerResumeConfig:
next_token: str | None = None
completed: bool = False
+ # (page token, child key) of the parent row being expanded at each fan-out level, outermost first.
+ parents: tuple[tuple[str | None, str], ...] = ()
+
+ def __post_init__(self) -> None:
+ # Saved state comes back from JSON with lists in place of tuples.
+ object.__setattr__(self, "parents", tuple((token, key) for token, key in self.parents))
class SystemsManagerError(Exception):
@@ -131,6 +155,45 @@ def request_payload(endpoint: SystemsManagerEndpoint, *, probe: bool = False) ->
return payload
+def endpoint_chain(endpoint: SystemsManagerEndpoint) -> list[SystemsManagerEndpoint]:
+ chain = [endpoint]
+ while chain[0].parent is not None:
+ chain.insert(0, ENDPOINTS[chain[0].parent])
+ return chain
+
+
+def _instance_status(item: dict[str, Any]) -> str | None:
+ data = item.get("Data")
+ info = data.get("AWS:InstanceInformation") if isinstance(data, dict) else None
+ content = info.get("Content") if isinstance(info, dict) else None
+ if isinstance(content, list) and content and isinstance(content[0], dict):
+ return content[0].get("InstanceStatus")
+ return None
+
+
+def child_payloads(
+ endpoint: SystemsManagerEndpoint, parent: dict[str, Any], inventory_types: Sequence[str]
+) -> list[dict[str, Any]]:
+ if endpoint.operation == "ListComplianceItems":
+ return [
+ {
+ "ResourceIds": [parent["ResourceId"]],
+ "ResourceTypes": [parent["ResourceType"]],
+ "Filters": [{"Key": "ComplianceType", "Values": [parent["ComplianceType"]], "Type": "EQUAL"}],
+ }
+ ]
+ if endpoint.operation == "ListInventoryEntries":
+ # ListInventoryEntries rejects terminated nodes, which GetInventory still lists.
+ if _instance_status(parent) == "Terminated":
+ return []
+ return [{"InstanceId": parent["Id"], "TypeName": type_name} for type_name in inventory_types]
+ if endpoint.operation == "DescribeAssociationExecutions":
+ return [{"AssociationId": parent["AssociationId"]}]
+ if endpoint.operation == "DescribeAssociationExecutionTargets":
+ return [{"AssociationId": parent["AssociationId"], "ExecutionId": parent["ExecutionId"]}]
+ raise ValueError(f"AWS Systems Manager table has no parent mapping: {endpoint.operation}")
+
+
def _flatten(obj: dict[str, Any], prefix: str = "") -> dict[str, Any]:
row: dict[str, Any] = {}
for key, value in obj.items():
@@ -155,6 +218,156 @@ def normalize_row(item: dict[str, Any], endpoint: SystemsManagerEndpoint, region
return row
+def page_rows(
+ endpoint: SystemsManagerEndpoint, body: dict[str, Any], payload: dict[str, Any], region: str
+) -> list[dict[str, Any]]:
+ items = body.get(endpoint.result_key) or []
+ if endpoint.operation != "ListInventoryEntries":
+ return [normalize_row(item, endpoint, region) for item in items]
+ capture_time = body.get("CaptureTime")
+ rows: list[dict[str, Any]] = []
+ for item in items:
+ # Entry keys depend on the inventory type, and entries carry no identifier of their own.
+ entry = json.dumps(item, sort_keys=True)
+ rows.append(
+ {
+ "region": region,
+ "instance_id": payload["InstanceId"],
+ "type_name": payload["TypeName"],
+ "schema_version": body.get("SchemaVersion"),
+ "capture_time": dt.datetime.fromisoformat(capture_time) if isinstance(capture_time, str) else None,
+ "entry_id": hashlib.sha256(entry.encode()).hexdigest(),
+ "entry": entry,
+ }
+ )
+ return rows
+
+
+def _next_token(body: dict[str, Any], token: str | None) -> str | None:
+ next_token = body.get("NextToken") or None
+ if next_token is not None and (not isinstance(next_token, str) or next_token == token):
+ raise ValueError("AWS Systems Manager returned an invalid or repeated page token.")
+ return next_token
+
+
+class SystemsManagerWalk:
+ """Walks an endpoint and its fan-out parents, yielding each page with the state that resumes after it."""
+
+ def __init__(
+ self,
+ client: SystemsManagerClient,
+ chain: list[SystemsManagerEndpoint],
+ manager: ResumableSourceManager[SystemsManagerResumeConfig],
+ ) -> None:
+ self._client = client
+ self._chain = chain
+ self._manager = manager
+ self._inventory_types: tuple[str, ...] = ()
+
+ def pages(
+ self, state: SystemsManagerResumeConfig | None
+ ) -> Iterator[tuple[list[dict[str, Any]], SystemsManagerResumeConfig]]:
+ if any(endpoint.operation == "ListInventoryEntries" for endpoint in self._chain):
+ self._inventory_types = self._load_inventory_types()
+ yield from self._walk(0, request_payload(self._chain[0]), (), state)
+
+ def _load_inventory_types(self) -> tuple[str, ...]:
+ names: set[str] = set()
+ token: str | None = None
+ while True:
+ payload: dict[str, Any] = {"MaxResults": 200}
+ if token:
+ payload["NextToken"] = token
+ body = self._client.request("GetInventorySchema", payload)
+ names.update(
+ schema["TypeName"]
+ for schema in body.get("Schemas") or []
+ if isinstance(schema, dict) and isinstance(schema.get("TypeName"), str)
+ )
+ self._manager.safe_point()
+ token = _next_token(body, token)
+ if token is None:
+ return tuple(sorted(names - EXCLUDED_INVENTORY_TYPES))
+
+ def _fetch(
+ self,
+ endpoint: SystemsManagerEndpoint,
+ payload: dict[str, Any],
+ token: str | None,
+ *,
+ resumed: bool,
+ child: bool,
+ ) -> tuple[dict[str, Any], str | None]:
+ while True:
+ try:
+ return self._client.request(
+ endpoint.operation, {**payload, "NextToken": token} if token else payload
+ ), token
+ except SystemsManagerError as error:
+ if resumed and token and error.code == "InvalidNextToken":
+ token, resumed = None, False
+ continue
+ if child and error.code in MISSING_PARENT_ERRORS:
+ return {}, token
+ raise
+
+ def _walk(
+ self,
+ level: int,
+ payload: dict[str, Any],
+ prefix: tuple[tuple[str | None, str], ...],
+ state: SystemsManagerResumeConfig | None,
+ ) -> Iterator[tuple[list[dict[str, Any]], SystemsManagerResumeConfig]]:
+ if state is not None and state.completed and not state.parents:
+ return
+ endpoint = self._chain[level]
+ if level == len(self._chain) - 1:
+ token = state.next_token if state else None
+ resumed = token is not None
+ while True:
+ body, token = self._fetch(endpoint, payload, token, resumed=resumed, child=level > 0)
+ resumed = False
+ next_token = _next_token(body, token)
+ rows = page_rows(endpoint, body, payload, self._client.region)
+ yield (
+ rows,
+ SystemsManagerResumeConfig(parents=prefix, next_token=next_token, completed=next_token is None),
+ )
+ if next_token is None:
+ return
+ token = next_token
+
+ token, key = state.parents[0] if state and state.parents else (None, "")
+ inner = dataclasses.replace(state, parents=state.parents[1:]) if state and state.parents else None
+ resumed = token is not None
+ child_endpoint = self._chain[level + 1]
+ while True:
+ body, used = self._fetch(endpoint, payload, token, resumed=resumed, child=level > 0)
+ if used != token:
+ key, inner = "", None
+ token, resumed = used, False
+ next_token = _next_token(body, token)
+ children = {
+ json.dumps(child, sort_keys=True): child
+ for item in body.get(endpoint.result_key) or []
+ for child in child_payloads(child_endpoint, item, self._inventory_types)
+ }
+ keys = list(children)
+ start = keys.index(key) if key in children else 0
+ for child_key in keys[start:]:
+ yield from self._walk(
+ level + 1,
+ {"MaxResults": child_endpoint.page_size, **children[child_key]},
+ (*prefix, (token, child_key)),
+ inner if child_key == key else None,
+ )
+ if next_token is None:
+ yield [], SystemsManagerResumeConfig(parents=prefix, completed=True)
+ return
+ token, key, inner = next_token, "", None
+ yield [], SystemsManagerResumeConfig(parents=(*prefix, (token, "")))
+
+
def get_rows(
config: AwsSystemsManagerSourceConfig,
endpoint: SystemsManagerEndpoint,
@@ -162,40 +375,30 @@ def get_rows(
manager: ResumableSourceManager[SystemsManagerResumeConfig],
) -> Iterator[list[dict[str, Any]]]:
state = manager.load_state() if manager.can_resume() else None
- if state and state.completed:
+ if state and state.completed and not state.parents:
return
- next_token = state.next_token if state else None
- resumed = bool(next_token)
client = SystemsManagerClient(config, api_version)
try:
- while True:
- payload = request_payload(endpoint)
- if next_token:
- payload["NextToken"] = next_token
- try:
- body = client.request(endpoint.operation, payload)
- except SystemsManagerError as error:
- if resumed and error.code == "InvalidNextToken":
- next_token = None
- resumed = False
- continue
- raise
- resumed = False
- rows = [normalize_row(item, endpoint, client.region) for item in body.get(endpoint.result_key, [])]
- token = body.get("NextToken") or None
- if token is not None and (not isinstance(token, str) or token == next_token):
- raise ValueError("AWS Systems Manager returned an invalid or repeated page token.")
- next_token = token
- manager.save_state(SystemsManagerResumeConfig(next_token=next_token, completed=next_token is None))
+ for rows, next_state in SystemsManagerWalk(client, endpoint_chain(endpoint), manager).pages(state):
+ manager.save_state(next_state)
if rows:
yield rows
manager.safe_point()
- if next_token is None:
- return
finally:
client.close()
+def probe_payload(child: SystemsManagerEndpoint, parent_body: dict[str, Any]) -> dict[str, Any]:
+ # IAM can scope the child operations to specific resources, so probe a real parent row when one exists.
+ parent_endpoint = ENDPOINTS[child.parent or ""]
+ for item in (parent_body.get(parent_endpoint.result_key) or [])[:1]:
+ payloads = child_payloads(child, item, ("AWS:Application",))
+ if payloads:
+ return {**payloads[0], "MaxResults": child.probe_size}
+ payload = child_payloads(child, PLACEHOLDER_PARENTS[child.parent or ""], ("AWS:Application",))[0]
+ return {**payload, "MaxResults": child.probe_size}
+
+
def validate_credentials(
config: AwsSystemsManagerSourceConfig, api_version: str, schema_name: str | None = None
) -> tuple[bool, str | None]:
@@ -208,13 +411,26 @@ def validate_credentials(
client = SystemsManagerClient(config, api_version)
except ValueError as error:
return False, str(error)
+ chain = endpoint_chain(endpoint)
+ operation = chain[0].operation
try:
- client.request(endpoint.operation, request_payload(endpoint, probe=True))
+ body = client.request(operation, request_payload(chain[0], probe=True))
+ if endpoint.operation == "ListInventoryEntries":
+ operation = "GetInventorySchema"
+ client.request(operation, {"MaxResults": 50})
+ for child in chain[1:]:
+ operation = child.operation
+ try:
+ body = client.request(operation, probe_payload(child, body))
+ except SystemsManagerError as error:
+ if error.code not in MISSING_PARENT_ERRORS:
+ raise
+ body = {}
except SystemsManagerError as error:
if error.code in {"AccessDenied", "AccessDeniedException"}:
if schema_name is None:
return True, None
- return False, f"Grant ssm:{endpoint.operation} to read this table."
+ return False, f"Grant ssm:{operation} to read this table."
return False, ERROR_MESSAGES.get(
error.code, "AWS Systems Manager could not validate the credentials. Try again."
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/canonical_descriptions.py
index 3e63f5f0ff47..a28f83d88920 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/canonical_descriptions.py
@@ -70,4 +70,84 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"operating_system": "Operating system supported by the patch baseline.",
},
},
+ "documents": {
+ "description": "SSM documents available in the selected region, including documents provided by AWS.",
+ "docs_url": "https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_ListDocuments.html",
+ "columns": {
+ "region": "AWS region selected for this source.",
+ "name": "Name of the SSM document. Associations refer to documents by this name.",
+ "owner": "AWS account that created the document, or Amazon for documents that AWS provides.",
+ "display_name": "Friendly name of the document.",
+ "document_type": "Type of document, such as Command, Automation, or Policy.",
+ "document_version": "Default version of the document.",
+ "version_name": "Optional name of the default document version.",
+ "document_format": "Format of the document content, such as JSON or YAML.",
+ "schema_version": "Schema version of the document.",
+ "platform_types": "Operating systems that the document supports.",
+ "target_type": "Resource type that the document can run on.",
+ "created_date": "Time when the document was created.",
+ "review_status": "Review status of the current document version.",
+ "author": "User who created the document.",
+ },
+ },
+ "compliance_items": {
+ "description": "Individual compliance results for each resource, such as one row per patch or association.",
+ "docs_url": "https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_ListComplianceItems.html",
+ "columns": {
+ "region": "AWS region selected for this source.",
+ "resource_id": "Identifier of the resource checked for compliance.",
+ "resource_type": "Type of resource checked for compliance.",
+ "compliance_type": "Compliance category, such as Patch, Association, or a Custom type.",
+ "id": "Identifier of the compliance item, such as a patch KB article or an association ID.",
+ "title": "Title of the compliance item, such as the patch title.",
+ "status": "Compliance status of the item: COMPLIANT or NON_COMPLIANT.",
+ "severity": "Severity of the compliance item.",
+ "execution_summary_execution_time": "Time of the execution that reported this item.",
+ "execution_summary_execution_id": "Identifier of the execution that reported this item.",
+ "execution_summary_execution_type": "Type of execution that reported this item.",
+ },
+ },
+ "inventory_entries": {
+ "description": "Inventory entries collected from each managed node, one row per entry, for every inventory type except AWS:InstanceInformation.",
+ "docs_url": "https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_ListInventoryEntries.html",
+ "columns": {
+ "region": "AWS region selected for this source.",
+ "instance_id": "Identifier of the managed node that reported the entry.",
+ "type_name": "Inventory type of the entry, such as AWS:Application or AWS:Service.",
+ "schema_version": "Inventory schema version used by the managed node.",
+ "capture_time": "Time when the node collected the inventory.",
+ "entry_id": "SHA-256 hash of the entry content. Entries have no identifier of their own.",
+ "entry": "JSON object with the entry attributes. Keys depend on the inventory type, for example Name and Version for applications.",
+ },
+ },
+ "association_executions": {
+ "description": "Execution history of State Manager associations.",
+ "docs_url": "https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_DescribeAssociationExecutions.html",
+ "columns": {
+ "region": "AWS region selected for this source.",
+ "association_id": "Identifier of the association.",
+ "association_version": "Version of the association that ran.",
+ "execution_id": "Identifier of the execution.",
+ "status": "Status of the execution.",
+ "detailed_status": "Detailed status of the execution.",
+ "created_time": "Time when the execution started.",
+ "last_execution_date": "Time when the association last ran for this execution.",
+ "resource_count_by_status": "Number of targeted resources in each status, such as {Success=3}.",
+ },
+ },
+ "association_execution_targets": {
+ "description": "Resources targeted by each association execution, with the result for each resource.",
+ "docs_url": "https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_DescribeAssociationExecutionTargets.html",
+ "columns": {
+ "region": "AWS region selected for this source.",
+ "association_id": "Identifier of the association.",
+ "association_version": "Version of the association that ran.",
+ "execution_id": "Identifier of the execution.",
+ "resource_id": "Identifier of the targeted resource.",
+ "resource_type": "Type of the targeted resource.",
+ "status": "Status of the execution on this resource.",
+ "detailed_status": "Detailed status of the execution on this resource.",
+ "last_execution_date": "Time when the execution last ran on this resource.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/settings.py
index e51b4edd1e61..d1f1f38e7afc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/settings.py
@@ -15,6 +15,8 @@ class SystemsManagerEndpoint:
page_size: int = 50
probe_size: int = 1
timestamps: tuple[str, ...] = ()
+ # Endpoint whose rows supply the required identifiers for each request to this one.
+ parent: str | None = None
ENDPOINTS: dict[str, SystemsManagerEndpoint] = {
@@ -58,8 +60,54 @@ ENDPOINTS: dict[str, SystemsManagerEndpoint] = {
description="Patch baseline identities, operating systems, and default baseline flags.",
page_size=100,
),
+ "documents": SystemsManagerEndpoint(
+ operation="ListDocuments",
+ result_key="DocumentIdentifiers",
+ primary_key=("region", "owner", "name"),
+ description="SSM documents owned by you, shared with you, or provided by AWS, with types, versions, and platforms.",
+ timestamps=("created_date",),
+ ),
+ "compliance_items": SystemsManagerEndpoint(
+ operation="ListComplianceItems",
+ result_key="ComplianceItems",
+ primary_key=("region", "resource_type", "resource_id", "compliance_type", "id"),
+ description="Individual patch, association, and custom compliance results behind each resource compliance summary.",
+ timestamps=("execution_summary_execution_time",),
+ parent="resource_compliance_summaries",
+ ),
+ "inventory_entries": SystemsManagerEndpoint(
+ operation="ListInventoryEntries",
+ result_key="Entries",
+ primary_key=("region", "instance_id", "type_name", "entry_id"),
+ description="Inventory entries of every collected type except AWS:InstanceInformation, such as applications and services, for each node.",
+ parent="inventory",
+ ),
+ "association_executions": SystemsManagerEndpoint(
+ operation="DescribeAssociationExecutions",
+ result_key="AssociationExecutions",
+ primary_key=("region", "association_id", "execution_id"),
+ description="Execution history of each State Manager association, with status and resource counts by status.",
+ timestamps=("created_time", "last_execution_date"),
+ parent="associations",
+ ),
+ "association_execution_targets": SystemsManagerEndpoint(
+ operation="DescribeAssociationExecutionTargets",
+ result_key="AssociationExecutionTargets",
+ primary_key=("region", "association_id", "execution_id", "resource_type", "resource_id"),
+ description="Status of each resource targeted by an association execution.",
+ timestamps=("last_execution_date",),
+ parent="association_executions",
+ ),
}
+# Other tables already sync these inventory types.
+EXCLUDED_INVENTORY_TYPES = frozenset({"AWS:InstanceInformation", "AWS:ComplianceItem", "AWS:ComplianceSummary"})
+
+# A parent can disappear between listing it and reading its children.
+MISSING_PARENT_ERRORS = frozenset(
+ {"AssociationDoesNotExist", "AssociationExecutionDoesNotExist", "InvalidInstanceId", "InvalidResourceId"}
+)
+
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}
ERROR_MESSAGES = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/source.py
index 4a7001927524..1128930260f4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/source.py
@@ -111,11 +111,13 @@ class AwsSystemsManagerSource(ResumableSource[AwsSystemsManagerSourceConfig, Sys
name=ExternalDataSourceType.AWSSYSTEMSMANAGER,
category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,
label="AWS Systems Manager",
- caption="""Sync fleet inventory, compliance summaries, associations, and patch baselines from one AWS region.
+ caption="""Sync fleet inventory, compliance, associations, documents, and patch baselines from one AWS region.
Grant these IAM permissions for the tables you select:
-`ssm:DescribeInstanceInformation`, `ssm:GetInventory`, `ssm:ListResourceComplianceSummaries`,
-`ssm:ListAssociations`, and `ssm:DescribePatchBaselines`.
+`ssm:DescribeInstanceInformation`, `ssm:GetInventory`, `ssm:GetInventorySchema`, `ssm:ListInventoryEntries`,
+`ssm:ListResourceComplianceSummaries`, `ssm:ListComplianceItems`, `ssm:ListAssociations`,
+`ssm:DescribeAssociationExecutions`, `ssm:DescribeAssociationExecutionTargets`, `ssm:ListDocuments`,
+and `ssm:DescribePatchBaselines`.
Enter an access key ID and secret access key. Include a session token for temporary credentials.
All tables use full refresh. Create a separate source for each region.""",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/tests/test_aws_systems_manager.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/tests/test_aws_systems_manager.py
index fc1fa109bc38..563cd7848fdf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/tests/test_aws_systems_manager.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_systems_manager/tests/test_aws_systems_manager.py
@@ -475,3 +475,311 @@ def test_snapshot_requests_never_filter_by_watermark(
)
list(iter_batches(AwsSystemsManagerSource().source_for_pipeline(config, manager, inputs)))
assert json.loads(http.post.call_args.kwargs["data"]) == {"MaxResults": 50}
+
+
+def _error(code: str) -> ScriptedResponse:
+ return ScriptedResponse(status=400, json={"__type": code, "Message": "Not found"})
+
+
+def _key(payload: dict[str, Any]) -> str:
+ return json.dumps(payload, sort_keys=True)
+
+
+def _instance(instance_id: str, status: str) -> dict[str, Any]:
+ return {"Id": instance_id, "Data": {"AWS:InstanceInformation": {"Content": [{"InstanceStatus": status}]}}}
+
+
+ "schema,script,expected_requests,expected_rows",
+ [
+ (
+ "compliance_items",
+ [
+ ScriptedResponse(
+ json={
+ "ResourceComplianceSummaryItems": [
+ {"ResourceId": "i-a", "ResourceType": "ManagedInstance", "ComplianceType": "Patch"},
+ {"ResourceId": "i-gone", "ResourceType": "ManagedInstance", "ComplianceType": "Patch"},
+ ]
+ }
+ ),
+ ScriptedResponse(
+ json={
+ "ComplianceItems": [
+ {
+ "ResourceId": "i-a",
+ "ResourceType": "ManagedInstance",
+ "ComplianceType": "Patch",
+ "Id": "KB1",
+ "Status": "NON_COMPLIANT",
+ "Details": {"PatchState": "Missing"},
+ "ExecutionSummary": {"ExecutionTime": 1735689600},
+ }
+ ]
+ }
+ ),
+ _error("InvalidResourceId"),
+ ],
+ [
+ ("ListResourceComplianceSummaries", {"MaxResults": 50}),
+ *(
+ (
+ "ListComplianceItems",
+ {
+ "MaxResults": 50,
+ "ResourceIds": [resource_id],
+ "ResourceTypes": ["ManagedInstance"],
+ "Filters": [{"Key": "ComplianceType", "Values": ["Patch"], "Type": "EQUAL"}],
+ },
+ )
+ for resource_id in ["i-a", "i-gone"]
+ ),
+ ],
+ [
+ {
+ "resource_id": "i-a",
+ "id": "KB1",
+ "details_patch_state": "Missing",
+ "execution_summary_execution_time": dt.datetime(2025, 1, 1, tzinfo=dt.UTC),
+ }
+ ],
+ ),
+ (
+ "inventory_entries",
+ [
+ ScriptedResponse(
+ json={
+ "Schemas": [
+ {"TypeName": name} for name in ["AWS:Service", "AWS:InstanceInformation", "AWS:Application"]
+ ]
+ }
+ ),
+ ScriptedResponse(
+ json={"Entities": [_instance("i-0123456789abcdef0", "Active"), _instance("i-ended", "Terminated")]}
+ ),
+ ScriptedResponse(
+ json={
+ "InstanceId": "i-0123456789abcdef0",
+ "TypeName": "AWS:Application",
+ "SchemaVersion": "1.1",
+ "CaptureTime": "2025-01-01T00:00:00Z",
+ "Entries": [{"Name": "nginx", "Version": "1.0"}, {"Name": "curl", "Version": "8.0"}],
+ }
+ ),
+ _error("InvalidInstanceId"),
+ ],
+ [
+ ("GetInventorySchema", {"MaxResults": 200}),
+ ("GetInventory", {"MaxResults": 50, "ResultAttributes": [{"TypeName": "AWS:InstanceInformation"}]}),
+ *(
+ ("ListInventoryEntries", {"MaxResults": 50, "InstanceId": "i-0123456789abcdef0", "TypeName": name})
+ for name in ["AWS:Application", "AWS:Service"]
+ ),
+ ],
+ [
+ {
+ "instance_id": "i-0123456789abcdef0",
+ "type_name": "AWS:Application",
+ "capture_time": dt.datetime(2025, 1, 1, tzinfo=dt.UTC),
+ "entry": json.dumps({"Name": name, "Version": version}, sort_keys=True),
+ }
+ for name, version in [("nginx", "1.0"), ("curl", "8.0")]
+ ],
+ ),
+ (
+ "association_execution_targets",
+ [
+ ScriptedResponse(json={"Associations": [{"AssociationId": "a-1"}]}),
+ ScriptedResponse(
+ json={
+ "AssociationExecutions": [
+ {"AssociationId": "a-1", "ExecutionId": "e-1"},
+ {"AssociationId": "a-1", "ExecutionId": "e-gone"},
+ ]
+ }
+ ),
+ ScriptedResponse(
+ json={
+ "AssociationExecutionTargets": [
+ {
+ "AssociationId": "a-1",
+ "ExecutionId": "e-1",
+ "ResourceId": "i-a",
+ "ResourceType": "ManagedInstance",
+ "Status": "Failed",
+ "LastExecutionDate": 1735689600,
+ }
+ ]
+ }
+ ),
+ _error("AssociationExecutionDoesNotExist"),
+ ],
+ [
+ ("ListAssociations", {"MaxResults": 50}),
+ ("DescribeAssociationExecutions", {"MaxResults": 50, "AssociationId": "a-1"}),
+ *(
+ (
+ "DescribeAssociationExecutionTargets",
+ {"MaxResults": 50, "AssociationId": "a-1", "ExecutionId": execution_id},
+ )
+ for execution_id in ["e-1", "e-gone"]
+ ),
+ ],
+ [
+ {
+ "execution_id": "e-1",
+ "resource_id": "i-a",
+ "status": "Failed",
+ "last_execution_date": dt.datetime(2025, 1, 1, tzinfo=dt.UTC),
+ }
+ ],
+ ),
+ ],
+)
+def test_fan_out_requests_each_parent_row_and_skips_vanished_parents(
+ config: AwsSystemsManagerSourceConfig,
+ schema: str,
+ script: list[ScriptedResponse],
+ expected_requests: list[tuple[str, dict[str, Any]]],
+ expected_rows: list[dict[str, Any]],
+) -> None:
+ result = driver(config).run(schema, script)
+ assert result.raised is None
+ assert [
+ (request.headers["x-amz-target"].removeprefix("AmazonSSM."), request.json()) for request in result.requests
+ ] == expected_requests
+ assert [{key: row[key] for key in expected} for row, expected in zip(result.rows, expected_rows)] == expected_rows
+ assert len(result.rows) == len(expected_rows)
+ assert result.response is not None
+ keys = {tuple(row[column] for column in result.response.primary_keys or []) for row in result.rows}
+ assert len(keys) == len(result.rows) and all(None not in key for key in keys)
+ assert result.final_state == SystemsManagerResumeConfig(completed=True)
+
+
+def test_fan_out_resumes_inside_nested_parent_without_rereading_done_children(
+ config: AwsSystemsManagerSourceConfig,
+) -> None:
+ a1 = _key({"AssociationId": "a-1"})
+ e1 = _key({"AssociationId": "a-1", "ExecutionId": "e-1"})
+ target = {"AssociationId": "a-1", "ResourceId": "i-a", "ResourceType": "ManagedInstance"}
+ result = driver(config).run(
+ "association_execution_targets",
+ [
+ ScriptedResponse(
+ json={
+ "Associations": [{"AssociationId": "a-0"}, {"AssociationId": "a-1"}],
+ "NextToken": "associations-2",
+ }
+ ),
+ ScriptedResponse(
+ json={
+ "AssociationExecutions": [
+ {"AssociationId": "a-1", "ExecutionId": "e-0"},
+ {"AssociationId": "a-1", "ExecutionId": "e-1"},
+ {"AssociationId": "a-1", "ExecutionId": "e-2"},
+ ],
+ }
+ ),
+ ScriptedResponse(json={"AssociationExecutionTargets": [{**target, "ExecutionId": "e-1"}]}),
+ ScriptedResponse(json={"AssociationExecutionTargets": [{**target, "ExecutionId": "e-2"}]}),
+ ScriptedResponse(json={"Associations": []}),
+ ],
+ resume_state=SystemsManagerResumeConfig(parents=((None, a1), ("executions-2", e1)), next_token="targets-2"),
+ )
+ assert result.raised is None
+ assert [request.json() for request in result.requests] == [
+ {"MaxResults": 50},
+ {"MaxResults": 50, "AssociationId": "a-1", "NextToken": "executions-2"},
+ {"MaxResults": 50, "AssociationId": "a-1", "ExecutionId": "e-1", "NextToken": "targets-2"},
+ {"MaxResults": 50, "AssociationId": "a-1", "ExecutionId": "e-2"},
+ {"MaxResults": 50, "NextToken": "associations-2"},
+ ]
+ assert [row["execution_id"] for row in result.rows] == ["e-1", "e-2"]
+ assert result.final_state == SystemsManagerResumeConfig(completed=True)
+
+
+_PLACEHOLDER_UUID = "00000000-0000-0000-0000-000000000000"
+
+
+ "schema,script,expected_requests,expected",
+ [
+ (
+ "association_execution_targets",
+ [
+ ScriptedResponse(json={"Associations": [{"AssociationId": "a-1"}]}),
+ ScriptedResponse(json={"AssociationExecutions": [{"AssociationId": "a-1", "ExecutionId": "e-1"}]}),
+ ScriptedResponse(json={}),
+ ],
+ [
+ ("ListAssociations", {"MaxResults": 1}),
+ ("DescribeAssociationExecutions", {"MaxResults": 1, "AssociationId": "a-1"}),
+ (
+ "DescribeAssociationExecutionTargets",
+ {"MaxResults": 1, "AssociationId": "a-1", "ExecutionId": "e-1"},
+ ),
+ ],
+ (True, None),
+ ),
+ (
+ "association_execution_targets",
+ [ScriptedResponse(json={}), _error("AssociationDoesNotExist"), _error("AssociationDoesNotExist")],
+ [
+ ("ListAssociations", {"MaxResults": 1}),
+ ("DescribeAssociationExecutions", {"MaxResults": 1, "AssociationId": _PLACEHOLDER_UUID}),
+ (
+ "DescribeAssociationExecutionTargets",
+ {"MaxResults": 1, "AssociationId": _PLACEHOLDER_UUID, "ExecutionId": _PLACEHOLDER_UUID},
+ ),
+ ],
+ (True, None),
+ ),
+ (
+ "inventory_entries",
+ [ScriptedResponse(json={}), _error("AccessDeniedException")],
+ [
+ ("GetInventory", {"MaxResults": 1, "ResultAttributes": [{"TypeName": "AWS:InstanceInformation"}]}),
+ ("GetInventorySchema", {"MaxResults": 50}),
+ ],
+ (False, "Grant ssm:GetInventorySchema to read this table."),
+ ),
+ (
+ "compliance_items",
+ [
+ ScriptedResponse(
+ json={
+ "ResourceComplianceSummaryItems": [
+ {"ResourceId": "i-a", "ResourceType": "ManagedInstance", "ComplianceType": "Association"}
+ ]
+ }
+ ),
+ _error("AccessDeniedException"),
+ ],
+ [
+ ("ListResourceComplianceSummaries", {"MaxResults": 1}),
+ (
+ "ListComplianceItems",
+ {
+ "MaxResults": 1,
+ "ResourceIds": ["i-a"],
+ "ResourceTypes": ["ManagedInstance"],
+ "Filters": [{"Key": "ComplianceType", "Values": ["Association"], "Type": "EQUAL"}],
+ },
+ ),
+ ],
+ (False, "Grant ssm:ListComplianceItems to read this table."),
+ ),
+ ],
+)
+def test_fan_out_validation_probes_real_parents_before_placeholders(
+ config: AwsSystemsManagerSourceConfig,
+ schema: str,
+ script: list[ScriptedResponse],
+ expected_requests: list[tuple[str, dict[str, Any]]],
+ expected: tuple[bool, str | None],
+) -> None:
+ with scripted_network(script) as network:
+ assert validate_credentials(config, API_VERSION, schema) == expected
+ assert [
+ (request.headers["x-amz-target"].removeprefix("AmazonSSM."), request.json()) for request in network.requests_log
+ ] == expected_requests