Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add hierarchy and delegation tables to aws organizations (#117702)

Tom Owers38c6dfbe64be
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 1aef1e84f96f..edbda4711798 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1252,14 +1252,14 @@ Note: Compared the official Macie V2 botocore operation model with the static en
## AWS Organizations — gaps
-Today (6): `accounts`, `organization`, `organizational_units`, `policies`, `resource_tags`, `roots`
+Today (10): `account_parents`, `accounts`, `delegated_administrators`, `delegated_services`, `organization`, `organizational_units`, `policies`, `policy_targets`, `resource_tags`, `roots`
Diffed against: <https://raw.githubusercontent.com/boto/botocore/develop/botocore/data/organizations/2016-11-28/service-2.json>
-- [ ] `Account parents (ListParents, POST /)` — connects synced accounts to organizational units and roots for cost allocation by organizational structure (high)
-- [ ] `Policy targets (ListTargetsForPolicy, POST /)` — connects synced policies to accounts, organizational units, and roots for governance analysis (high)
-- [ ] `Delegated administrators (ListDelegatedAdministrators, POST /)` — identifies accounts responsible for administering organization services (medium)
-- [ ] `Delegated services for account (ListDelegatedServicesForAccount, POST /)` — connects delegated administrator accounts to the services they manage (medium)
+- [x] `Account parents (ListParents, POST /)` — connects synced accounts to organizational units and roots for cost allocation by organizational structure (high)
+- [x] `Policy targets (ListTargetsForPolicy, POST /)` — connects synced policies to accounts, organizational units, and roots for governance analysis (high)
+- [x] `Delegated administrators (ListDelegatedAdministrators, POST /)` — identifies accounts responsible for administering organization services (medium)
+- [x] `Delegated services for account (ListDelegatedServicesForAccount, POST /)` — connects delegated administrator accounts to the services they manage (medium)
Note: Compared the official botocore operation model with the static endpoint map and extraction code. Organizational units already include parent IDs; account rows do not include their parent organizational unit. Account parents, policy targets, and delegated services require traversal; JSON RPC reads use POST /. Tags are already synced; policy content, service setup, and account creation plumbing were excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/aws_organizations.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/aws_organizations.py
index 7f4482598f2b..7f6e8dd87c4b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/aws_organizations.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/aws_organizations.py
@@ -341,13 +341,12 @@ def _list_rows(
)
-def _policy_rows(
- client: AwsOrganizationsClient,
- endpoint_config: AwsOrganizationsEndpointConfig,
+def _per_policy_type(
resumable_source_manager: ResumableSourceManager[AwsOrganizationsResumeConfig],
logger: FilteringBoundLogger,
+ rows_for_policy_type: Callable[[str, Optional[str]], Iterator[list[dict[str, Any]]]],
) -> Iterator[list[dict[str, Any]]]:
- """One walk per policy type, since `Filter` is required and takes a single type."""
+ """One walk per policy type, since `ListPolicies` requires `Filter` and takes a single type."""
filters = list(POLICY_FILTERS)
start_token: Optional[str] = None
@@ -364,6 +363,24 @@ def _policy_rows(
logger.debug(f"Resuming policies from a saved policy type. policy_type={resumed_filter}")
for policy_filter in filters:
+ try:
+ yield from rows_for_policy_type(policy_filter, start_token)
+ except AwsOrganizationsError as error:
+ if is_policy_type_unavailable(error):
+ logger.debug(f"Skipping unavailable policy type. policy_type={policy_filter}")
+ start_token = None
+ continue
+ raise
+ start_token = None
+
+
+def _policy_rows(
+ client: AwsOrganizationsClient,
+ endpoint_config: AwsOrganizationsEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[AwsOrganizationsResumeConfig],
+ logger: FilteringBoundLogger,
+) -> Iterator[list[dict[str, Any]]]:
+ def rows_for_policy_type(policy_filter: str, start_token: Optional[str]) -> Iterator[list[dict[str, Any]]]:
pages = iter_pages(
client,
endpoint_config.operation,
@@ -372,20 +389,187 @@ def _policy_rows(
logger,
start_token=start_token,
)
- try:
- yield from _checkpointed_rows(
- pages,
- lambda items: [normalize_row(endpoint_config, item) for item in items],
- resumable_source_manager,
- work_key=policy_filter,
+ return _checkpointed_rows(
+ pages,
+ lambda items: [normalize_row(endpoint_config, item) for item in items],
+ resumable_source_manager,
+ work_key=policy_filter,
+ )
+
+ yield from _per_policy_type(resumable_source_manager, logger, rows_for_policy_type)
+
+
+def _fan_out_rows(
+ parent_pages: Iterator[Page],
+ child_pages: Callable[[dict[str, Any]], Iterator[list[dict[str, Any]]]],
+ resumable_source_manager: ResumableSourceManager[AwsOrganizationsResumeConfig],
+ work_key: Optional[str] = None,
+) -> Iterator[list[dict[str, Any]]]:
+ """Child rows for every parent, resumable on the parent walk's page token.
+
+ A parent page's children are yielded as one batch, so the cursor never sits mid-page and a
+ resumed full refresh can't append the same children twice. The buffered rows are outside the
+ staged cursor, so a safe point while they are held only re-reads them.
+ """
+ for parent_page in parent_pages:
+ rows: list[dict[str, Any]] = []
+ for parent in parent_page.items:
+ for child_rows in child_pages(parent):
+ rows.extend(child_rows)
+ resumable_source_manager.safe_point()
+ resumable_source_manager.save_state(
+ AwsOrganizationsResumeConfig(next_token=parent_page.next_token, work_key=work_key)
+ )
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+
+
+def _child_pages(
+ client: AwsOrganizationsClient,
+ endpoint_config: AwsOrganizationsEndpointConfig,
+ payload: dict[str, Any],
+ parent_columns: dict[str, Any],
+ skip_codes: frozenset[str],
+ logger: FilteringBoundLogger,
+) -> Iterator[list[dict[str, Any]]]:
+ try:
+ for child_page in iter_pages(
+ client,
+ endpoint_config.operation,
+ {**payload, "MaxResults": endpoint_config.page_size},
+ result_key(endpoint_config),
+ logger,
+ ):
+ yield [{**normalize_row(endpoint_config, item), **parent_columns} for item in child_page.items]
+ except AwsOrganizationsError as error:
+ if error.code in skip_codes:
+ logger.debug(f"Skipping parent removed mid-sync. endpoint={endpoint_config.name} parent={parent_columns}")
+ return
+ raise
+
+
+def _single_walk_pages(
+ client: AwsOrganizationsClient,
+ operation: str,
+ items_key: str,
+ resumable_source_manager: ResumableSourceManager[AwsOrganizationsResumeConfig],
+ logger: FilteringBoundLogger,
+) -> Iterator[Page]:
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ # Every saved state names the page after the one it covers, so a missing token means the
+ # previous attempt finished the walk and only stopped before clearing its state.
+ if resume is not None and not resume.next_token:
+ logger.debug(f"Saved state marks the walk complete. operation={operation}")
+ return iter(())
+ start_token = resume.next_token if resume is not None else None
+ return iter_pages(client, operation, {"MaxResults": MAX_RESULTS}, items_key, logger, start_token=start_token)
+
+
+def _account_parent_rows(
+ client: AwsOrganizationsClient,
+ endpoint_config: AwsOrganizationsEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[AwsOrganizationsResumeConfig],
+ logger: FilteringBoundLogger,
+) -> Iterator[list[dict[str, Any]]]:
+ def parents_of(account: dict[str, Any]) -> Iterator[list[dict[str, Any]]]:
+ account_id = account.get("Id")
+ if not isinstance(account_id, str) or not account_id:
+ return
+ for rows in _child_pages(
+ client,
+ endpoint_config,
+ {"ChildId": account_id},
+ {"account_id": account_id},
+ frozenset({"ChildNotFoundException"}),
+ logger,
+ ):
+ yield [
+ {"account_id": row["account_id"], "parent_id": row.get("id"), "parent_type": row.get("type")}
+ for row in rows
+ ]
+
+ accounts = _single_walk_pages(client, "ListAccounts", "Accounts", resumable_source_manager, logger)
+ yield from _fan_out_rows(accounts, parents_of, resumable_source_manager)
+
+
+def _policy_target_rows(
+ client: AwsOrganizationsClient,
+ endpoint_config: AwsOrganizationsEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[AwsOrganizationsResumeConfig],
+ logger: FilteringBoundLogger,
+) -> Iterator[list[dict[str, Any]]]:
+ def rows_for_policy_type(policy_filter: str, start_token: Optional[str]) -> Iterator[list[dict[str, Any]]]:
+ def targets_of(policy: dict[str, Any]) -> Iterator[list[dict[str, Any]]]:
+ policy_id = policy.get("Id")
+ if not isinstance(policy_id, str) or not policy_id:
+ return iter(())
+ return _child_pages(
+ client,
+ endpoint_config,
+ {"PolicyId": policy_id},
+ {"policy_id": policy_id, "policy_type": policy_filter},
+ frozenset({"PolicyNotFoundException"}),
+ logger,
)
- except AwsOrganizationsError as error:
- if is_policy_type_unavailable(error):
- logger.debug(f"Skipping unavailable policy type. policy_type={policy_filter}")
- start_token = None
- continue
- raise
- start_token = None
+
+ policies = iter_pages(
+ client,
+ "ListPolicies",
+ {"Filter": policy_filter, "MaxResults": MAX_RESULTS},
+ "Policies",
+ logger,
+ start_token=start_token,
+ )
+ return _fan_out_rows(policies, targets_of, resumable_source_manager, work_key=policy_filter)
+
+ yield from _per_policy_type(resumable_source_manager, logger, rows_for_policy_type)
+
+
+def _delegated_service_rows(
+ client: AwsOrganizationsClient,
+ endpoint_config: AwsOrganizationsEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[AwsOrganizationsResumeConfig],
+ logger: FilteringBoundLogger,
+) -> Iterator[list[dict[str, Any]]]:
+ def services_of(administrator: dict[str, Any]) -> Iterator[list[dict[str, Any]]]:
+ account_id = administrator.get("Id")
+ if not isinstance(account_id, str) or not account_id:
+ return iter(())
+ return _child_pages(
+ client,
+ endpoint_config,
+ {"AccountId": account_id},
+ {"account_id": account_id},
+ # The account was deregistered or left the organization after it was listed.
+ frozenset({"AccountNotRegisteredException", "AccountNotFoundException"}),
+ logger,
+ )
+
+ administrators = _single_walk_pages(
+ client, "ListDelegatedAdministrators", "DelegatedAdministrators", resumable_source_manager, logger
+ )
+ yield from _fan_out_rows(administrators, services_of, resumable_source_manager)
+
+
+WALKS_BY_ENDPOINT: dict[
+ str,
+ Callable[
+ [
+ AwsOrganizationsClient,
+ AwsOrganizationsEndpointConfig,
+ ResumableSourceManager[AwsOrganizationsResumeConfig],
+ FilteringBoundLogger,
+ ],
+ Iterator[list[dict[str, Any]]],
+ ],
+] = {
+ "account_parents": _account_parent_rows,
+ "delegated_services": _delegated_service_rows,
+ "policies": _policy_rows,
+ "policy_targets": _policy_target_rows,
+}
# Named fields instead of a 2/3-tuple return: semgrep's tuple-return-prefer-dataclass
@@ -546,8 +730,9 @@ def get_rows(
yield from _resource_tag_rows(client, endpoint_config, logger)
return
- if endpoint == "policies":
- yield from _policy_rows(client, endpoint_config, resumable_source_manager, logger)
+ resumable_rows = WALKS_BY_ENDPOINT.get(endpoint)
+ if resumable_rows is not None:
+ yield from resumable_rows(client, endpoint_config, resumable_source_manager, logger)
else:
yield from _list_rows(client, endpoint_config, resumable_source_manager, logger)
@@ -566,6 +751,31 @@ def _permission_reason(error: AwsOrganizationsError) -> Optional[str]:
return None
+def _probe_first_child(
+ client: AwsOrganizationsClient,
+ parent_operation: str,
+ parent_payload: dict[str, Any],
+ parent_key: str,
+ child_operation: str,
+ child_param: str,
+) -> None:
+ next_token: Optional[str] = None
+ while True:
+ payload = {**parent_payload, "MaxResults": 1}
+ if next_token:
+ payload["NextToken"] = next_token
+ parents = client.request(parent_operation, payload)
+ for parent in parents.get(parent_key) or []:
+ parent_id = parent.get("Id") if isinstance(parent, dict) else None
+ if isinstance(parent_id, str):
+ client.request(child_operation, {child_param: parent_id, "MaxResults": 1})
+ return
+ # AWS can return an empty page while more parents remain.
+ next_token = parents.get("NextToken") or None
+ if not next_token:
+ return
+
+
def endpoint_permission_reason(client: AwsOrganizationsClient, endpoint: str) -> Optional[str]:
"""Probe the calls a sync of this endpoint issues. `None` when reachable.
@@ -588,6 +798,26 @@ def endpoint_permission_reason(client: AwsOrganizationsClient, endpoint: str) ->
break
elif endpoint == "policies":
client.request("ListPolicies", {"Filter": POLICY_FILTERS[0], "MaxResults": 1})
+ elif endpoint == "account_parents":
+ _probe_first_child(client, "ListAccounts", {}, "Accounts", "ListParents", "ChildId")
+ elif endpoint == "policy_targets":
+ _probe_first_child(
+ client,
+ "ListPolicies",
+ {"Filter": POLICY_FILTERS[0]},
+ "Policies",
+ "ListTargetsForPolicy",
+ "PolicyId",
+ )
+ elif endpoint == "delegated_services":
+ _probe_first_child(
+ client,
+ "ListDelegatedAdministrators",
+ {},
+ "DelegatedAdministrators",
+ "ListDelegatedServicesForAccount",
+ "AccountId",
+ )
elif endpoint == "resource_tags":
accounts = client.request("ListAccounts", {"MaxResults": 1})
for account in accounts.get("Accounts") or []:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/canonical_descriptions.py
index c7b362fde307..6670b3a690f6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/canonical_descriptions.py
@@ -3,6 +3,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.can
)
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
+ "account_parents": {
+ "description": "The root or organizational unit that each account sits directly under. Join to accounts on account_id and to organizational_units or roots on parent_id.",
+ "docs_url": "https://docs.aws.amazon.com/organizations/latest/APIReference/API_ListParents.html",
+ "columns": {
+ "account_id": "The unique identifier (ID) of the account, a 12-digit number.",
+ "parent_id": "The unique identifier (ID) of the parent entity: a root ID starting with 'r-' or an organizational unit ID starting with 'ou-'.",
+ "parent_type": "The type of the parent entity: ROOT or ORGANIZATIONAL_UNIT.",
+ },
+ },
"accounts": {
"description": "Every AWS account in the organization. The dimension table that maps a 12-digit account ID to a human-readable account name.",
"docs_url": "https://docs.aws.amazon.com/organizations/latest/APIReference/API_Account.html",
@@ -17,6 +26,30 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"paths": "The paths of the account in the organization hierarchy, from the root down to the account.",
},
},
+ "delegated_administrators": {
+ "description": "Accounts in the organization that are designated as delegated administrators for an AWS service.",
+ "docs_url": "https://docs.aws.amazon.com/organizations/latest/APIReference/API_DelegatedAdministrator.html",
+ "columns": {
+ "id": "The unique identifier (ID) of the delegated administrator's account.",
+ "arn": "The Amazon Resource Name (ARN) of the delegated administrator's account.",
+ "email": "The email address that is associated with the delegated administrator's AWS account.",
+ "name": "The friendly name of the delegated administrator's account.",
+ "status": "The status of the delegated administrator's account in the organization.",
+ "state": "The lifecycle state of the delegated administrator's account in the organization.",
+ "joined_method": "The method by which the delegated administrator's account joined the organization.",
+ "joined_timestamp": "The date when the delegated administrator's account became a part of the organization.",
+ "delegation_enabled_date": "The date when the account was made a delegated administrator.",
+ },
+ },
+ "delegated_services": {
+ "description": "The AWS services for which each delegated administrator account is a delegated administrator. One row per account and service.",
+ "docs_url": "https://docs.aws.amazon.com/organizations/latest/APIReference/API_DelegatedService.html",
+ "columns": {
+ "account_id": "The unique identifier (ID) of the delegated administrator's account.",
+ "service_principal": "The name of the AWS service, typically in the form servicename.amazonaws.com.",
+ "delegation_enabled_date": "The date that the account became a delegated administrator for this service.",
+ },
+ },
"organization": {
"description": "The organization itself, including its management account and which functionality is enabled. One row.",
"docs_url": "https://docs.aws.amazon.com/organizations/latest/APIReference/API_Organization.html",
@@ -53,6 +86,18 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"aws_managed": "True when the policy is managed by AWS, which means it can be attached to entities but not edited.",
},
},
+ "policy_targets": {
+ "description": "The roots, organizational units, and accounts that each policy is attached to. One row per attachment.",
+ "docs_url": "https://docs.aws.amazon.com/organizations/latest/APIReference/API_PolicyTargetSummary.html",
+ "columns": {
+ "policy_id": "The unique identifier (ID) of the attached policy.",
+ "policy_type": "The type of the attached policy, for example SERVICE_CONTROL_POLICY or TAG_POLICY.",
+ "target_id": "The unique identifier (ID) of the policy target: a root, organizational unit, or account ID.",
+ "arn": "The Amazon Resource Name (ARN) of the policy target.",
+ "name": "The friendly name of the policy target.",
+ "type": "The type of the policy target: ROOT, ORGANIZATIONAL_UNIT, or ACCOUNT.",
+ },
+ },
"resource_tags": {
"description": "Tags attached to accounts, roots, organizational units, and policies. One row per tag.",
"docs_url": "https://docs.aws.amazon.com/organizations/latest/APIReference/API_ListTagsForResource.html",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/settings.py
index 4f96c9b60935..d96d458eda40 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/settings.py
@@ -89,6 +89,14 @@ class AwsOrganizationsEndpointConfig:
AWS_ORGANIZATIONS_ENDPOINTS: dict[str, AwsOrganizationsEndpointConfig] = {
+ "account_parents": AwsOrganizationsEndpointConfig(
+ name="account_parents",
+ operation="ListParents",
+ primary_key=["account_id", "parent_id"],
+ result_key="Parents",
+ page_size=MAX_RESULTS,
+ iam_actions=("organizations:ListAccounts", "organizations:ListParents"),
+ ),
"accounts": AwsOrganizationsEndpointConfig(
name="accounts",
operation="ListAccounts",
@@ -101,6 +109,27 @@ AWS_ORGANIZATIONS_ENDPOINTS: dict[str, AwsOrganizationsEndpointConfig] = {
drop_keys=frozenset({"Status"}),
iam_actions=("organizations:ListAccounts",),
),
+ "delegated_administrators": AwsOrganizationsEndpointConfig(
+ name="delegated_administrators",
+ operation="ListDelegatedAdministrators",
+ primary_key=["id"],
+ result_key="DelegatedAdministrators",
+ page_size=MAX_RESULTS,
+ timestamp_columns=("joined_timestamp", "delegation_enabled_date"),
+ iam_actions=("organizations:ListDelegatedAdministrators",),
+ ),
+ "delegated_services": AwsOrganizationsEndpointConfig(
+ name="delegated_services",
+ operation="ListDelegatedServicesForAccount",
+ primary_key=["account_id", "service_principal"],
+ result_key="DelegatedServices",
+ page_size=MAX_RESULTS,
+ timestamp_columns=("delegation_enabled_date",),
+ iam_actions=(
+ "organizations:ListDelegatedAdministrators",
+ "organizations:ListDelegatedServicesForAccount",
+ ),
+ ),
"organization": AwsOrganizationsEndpointConfig(
name="organization",
operation="DescribeOrganization",
@@ -127,6 +156,14 @@ AWS_ORGANIZATIONS_ENDPOINTS: dict[str, AwsOrganizationsEndpointConfig] = {
page_size=MAX_RESULTS,
iam_actions=("organizations:ListPolicies",),
),
+ "policy_targets": AwsOrganizationsEndpointConfig(
+ name="policy_targets",
+ operation="ListTargetsForPolicy",
+ primary_key=["policy_id", "target_id"],
+ result_key="Targets",
+ page_size=MAX_RESULTS,
+ iam_actions=("organizations:ListPolicies", "organizations:ListTargetsForPolicy"),
+ ),
"resource_tags": AwsOrganizationsEndpointConfig(
name="resource_tags",
operation="ListTagsForResource",
@@ -158,10 +195,14 @@ ENDPOINTS = tuple(AWS_ORGANIZATIONS_ENDPOINTS.keys())
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}
ENDPOINT_DESCRIPTIONS: dict[str, str] = {
+ "account_parents": "The root or organizational unit each account sits directly under. One row per account.",
"accounts": "Every AWS account in the organization, with its name, email, state, and the time it joined.",
+ "delegated_administrators": "Accounts registered as delegated administrators for an AWS service in the organization.",
+ "delegated_services": "The AWS services each delegated administrator account administers. One row per account and service.",
"organization": "The organization itself: its ID, feature set, and management account. One row per sync.",
"organizational_units": "Organizational units in the account hierarchy, each with the parent it sits under.",
"policies": "Policies in the organization, one row per policy, across every policy type the organization supports.",
+ "policy_targets": "The roots, organizational units, and accounts each policy is attached to. One row per attachment.",
"resource_tags": "Tags attached to accounts, roots, organizational units, and policies. One row per tag.",
"roots": "Roots of the organization, with the policy types enabled on each.",
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/source.py
index 828c00cef7cc..b289d227f89a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/source.py
@@ -68,7 +68,7 @@ class AwsOrganizationsSource(ResumableSource[AwsOrganizationsSourceConfig, AwsOr
"AWS Organizations request failed: SignatureDoesNotMatch": "AWS rejected the request signature. Please re-enter the secret access key.",
"AWS Organizations request failed: InvalidSignatureException": "AWS rejected the request signature. If you are using temporary credentials, the session token has expired.",
"AWS Organizations request failed: ExpiredTokenException": "The AWS session token has expired. Please reconnect with fresh credentials.",
- "AWS Organizations request failed: AccessDeniedException": f"These AWS credentials are missing AWS Organizations read permissions. Grant organizations:DescribeOrganization, organizations:ListAccounts, organizations:ListRoots, organizations:ListOrganizationalUnitsForParent, organizations:ListPolicies and organizations:ListTagsForResource to the IAM user or role. {_MANAGEMENT_ACCOUNT_HINT}",
+ "AWS Organizations request failed: AccessDeniedException": f"These AWS credentials are missing AWS Organizations read permissions. Grant organizations:DescribeOrganization, organizations:ListAccounts, organizations:ListParents, organizations:ListRoots, organizations:ListOrganizationalUnitsForParent, organizations:ListPolicies, organizations:ListTargetsForPolicy, organizations:ListTagsForResource, organizations:ListDelegatedAdministrators and organizations:ListDelegatedServicesForAccount to the IAM user or role. {_MANAGEMENT_ACCOUNT_HINT}",
"AWS Organizations request failed: AWSOrganizationsNotInUseException": "This AWS account isn't a member of an organization. Connect a key from an account in an AWS organization.",
"AWS access key ID and secret access key are required": "Enter both an AWS access key ID and a secret access key.",
}
@@ -150,7 +150,7 @@ class AwsOrganizationsSource(ResumableSource[AwsOrganizationsSourceConfig, AwsOr
label="AWS Organizations",
caption="""Sync your AWS organization structure into the PostHog Data warehouse, so cost and usage data can be read by account name, organizational unit, and tag.
-Create an IAM user or role with the `organizations:DescribeOrganization`, `organizations:ListAccounts`, `organizations:ListRoots`, `organizations:ListOrganizationalUnitsForParent`, `organizations:ListPolicies` and `organizations:ListTagsForResource` permissions, then paste its access key ID and secret access key. Add a session token too if you are using temporary credentials.
+Create an IAM user or role with the `organizations:DescribeOrganization`, `organizations:ListAccounts`, `organizations:ListParents`, `organizations:ListRoots`, `organizations:ListOrganizationalUnitsForParent`, `organizations:ListPolicies`, `organizations:ListTargetsForPolicy`, `organizations:ListTagsForResource`, `organizations:ListDelegatedAdministrators` and `organizations:ListDelegatedServicesForAccount` permissions, then paste its access key ID and secret access key. Add a session token too if you are using temporary credentials.
Use a key from the management account or from a member account that is a delegated administrator. Keys from other member accounts can only read the organization table.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/tests/test_aws_organizations.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/tests/test_aws_organizations.py
index bc1e7d438bb6..d07612cc5c29 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/tests/test_aws_organizations.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/tests/test_aws_organizations.py
@@ -284,6 +284,112 @@ class TestResourceTags:
]
+class TestFanOut:
+ @pytest.mark.parametrize(
+ "endpoint,parent_operation,parent_key,child_operation,child_param,child_key,child_item,missing_code,parent_column",
+ [
+ (
+ "account_parents",
+ "ListAccounts",
+ "Accounts",
+ "ListParents",
+ "ChildId",
+ "Parents",
+ {"Id": "ou-exam-aaaaaaaa", "Type": "ORGANIZATIONAL_UNIT"},
+ "ChildNotFoundException",
+ "account_id",
+ ),
+ (
+ "policy_targets",
+ "ListPolicies",
+ "Policies",
+ "ListTargetsForPolicy",
+ "PolicyId",
+ "Targets",
+ {"TargetId": "111111111111", "Type": "ACCOUNT", "Name": "prod"},
+ "PolicyNotFoundException",
+ "policy_id",
+ ),
+ (
+ "delegated_services",
+ "ListDelegatedAdministrators",
+ "DelegatedAdministrators",
+ "ListDelegatedServicesForAccount",
+ "AccountId",
+ "DelegatedServices",
+ {"ServicePrincipal": "guardduty.amazonaws.com", "DelegationEnabledDate": JAN_2025},
+ "AccountNotRegisteredException",
+ "account_id",
+ ),
+ ],
+ )
+ def test_every_child_page_is_kept_with_its_parent_and_a_parent_removed_mid_sync_is_skipped(
+ self,
+ endpoint: str,
+ parent_operation: str,
+ parent_key: str,
+ child_operation: str,
+ child_param: str,
+ child_key: str,
+ child_item: dict[str, Any],
+ missing_code: str,
+ parent_column: str,
+ ) -> None:
+ def responder(request: RecordedRequest) -> ScriptedResponse:
+ payload = request.json()
+ operation = request_operation(request)
+ if operation == parent_operation:
+ if endpoint == "policy_targets" and payload["Filter"] != "SERVICE_CONTROL_POLICY":
+ return ScriptedResponse(json=page(parent_key, []))
+ if payload.get("NextToken") is None:
+ return ScriptedResponse(json=page(parent_key, [{"Id": "parent-a"}], next_token="parents-2"))
+ return ScriptedResponse(json=page(parent_key, [{"Id": "parent-gone"}, {"Id": "parent-b"}]))
+ assert operation == child_operation
+ if payload[child_param] == "parent-gone":
+ return aws_error(missing_code, "The entity no longer exists.")
+ if payload[child_param] == "parent-a" and payload.get("NextToken") is None:
+ return ScriptedResponse(json=page(child_key, [child_item], next_token="children-2"))
+ return ScriptedResponse(json=page(child_key, [child_item]))
+
+ result = driver().run(endpoint, responder)
+
+ assert result.raised is None
+ assert [[row[parent_column] for row in item] for item in result.items] == [
+ ["parent-a", "parent-a"],
+ ["parent-b"],
+ ]
+ child_payloads = [
+ request.json() for request in result.requests if request_operation(request) == child_operation
+ ]
+ assert all(payload["MaxResults"] == MAX_RESULTS for payload in child_payloads)
+
+ def test_account_parents_rows_name_the_parent_and_resume_the_account_walk(self) -> None:
+ def responder(request: RecordedRequest) -> ScriptedResponse:
+ if request_operation(request) == "ListAccounts":
+ return ScriptedResponse(json=page("Accounts", [{"Id": "222222222222"}]))
+ return ScriptedResponse(json=page("Parents", [{"Id": "r-exam", "Type": "ROOT"}]))
+
+ result = driver().run(
+ "account_parents", responder, resume_state=AwsOrganizationsResumeConfig(next_token="accounts-2")
+ )
+
+ assert result.raised is None
+ assert result.requests[0].json() == {"MaxResults": MAX_RESULTS, "NextToken": "accounts-2"}
+ assert result.rows == [{"account_id": "222222222222", "parent_id": "r-exam", "parent_type": "ROOT"}]
+
+ @pytest.mark.parametrize("endpoint", ["account_parents", "delegated_services"])
+ def test_a_walk_saved_as_complete_is_not_read_again(self, endpoint: str) -> None:
+ result = driver().run(
+ endpoint,
+ always(aws_error("ServiceException", "unexpected request")),
+ resume_state=AwsOrganizationsResumeConfig(next_token=None),
+ )
+
+ assert result.raised is None
+ assert result.requests == []
+ assert result.rows == []
+
+
class TestResume:
def test_an_expired_saved_token_restarts_the_walk_instead_of_failing(self) -> None:
def responder(request: RecordedRequest) -> ScriptedResponse:
@@ -310,9 +416,10 @@ class TestResume:
assert result.raised is None
assert [request.json()["Filter"] for request in result.requests] == list(POLICY_FILTERS[1:])
- def test_policies_resume_continues_a_half_walked_policy_type(self) -> None:
+ @pytest.mark.parametrize("endpoint", ["policies", "policy_targets"])
+ def test_policies_resume_continues_a_half_walked_policy_type(self, endpoint: str) -> None:
result = driver().run(
- "policies",
+ endpoint,
always(ScriptedResponse(json=page("Policies", [{"Id": "p-example"}]))),
resume_state=AwsOrganizationsResumeConfig(next_token="page-two", work_key=POLICY_FILTERS[1]),
)
@@ -322,7 +429,9 @@ class TestResume:
"ListPolicies",
{"Filter": POLICY_FILTERS[1], "MaxResults": MAX_RESULTS, "NextToken": "page-two"},
)
- assert [request.json()["Filter"] for request in result.requests] == list(POLICY_FILTERS[1:])
+ assert [
+ request.json()["Filter"] for request in result.requests if request_operation(request) == "ListPolicies"
+ ] == list(POLICY_FILTERS[1:])
class TestValidateCredentials:
@@ -384,7 +493,11 @@ class TestProbeEndpointPermissions:
match request_operation(request):
case "ListPolicies":
return aws_error("AccessDeniedException", "not authorized to perform organizations:ListPolicies")
+ case "ListParents":
+ return aws_error("AccessDeniedException", "not authorized to perform organizations:ListParents")
case "ListAccounts":
+ if request.json().get("NextToken") is None:
+ return ScriptedResponse(json=page("Accounts", [], next_token="accounts-2"))
return ScriptedResponse(json=page("Accounts", [{"Id": "111111111111"}]))
case "ListRoots":
return ScriptedResponse(json=page("Roots", [{"Id": "r-exam"}]))
@@ -395,10 +508,15 @@ class TestProbeEndpointPermissions:
with scripted_network(responder) as network:
reasons = probe_endpoint_permissions(
- "AKIAEXAMPLE", "secret", None, ["accounts", "organization", "organizational_units", "policies"]
+ "AKIAEXAMPLE",
+ "secret",
+ None,
+ ["accounts", "organization", "organizational_units", "policies", "account_parents"],
)
assert reasons["accounts"] is None
+ assert reasons["account_parents"] is not None
+ assert "organizations:ListParents" in reasons["account_parents"]
assert reasons["organization"] is None
assert reasons["organizational_units"] is None
assert reasons["policies"] is not None
@@ -409,6 +527,9 @@ class TestProbeEndpointPermissions:
"ListRoots",
"ListOrganizationalUnitsForParent",
"ListPolicies",
+ "ListAccounts",
+ "ListAccounts",
+ "ListParents",
]
@pytest.mark.parametrize("error", [AwsOrganizationsRetryableError("TooManyRequestsException", "slow down")])