:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add hierarchy and delegation tables to aws organizations (#117702)
Tom Owers38c6dfbe64be
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 1aef1e84f96f..edbda4711798 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -1252,14 +1252,14 @@ Note: Compared the official Macie V2 botocore operation model with the static en## AWS Organizations — gaps-Today (6): `accounts`, `organization`, `organizational_units`, `policies`, `resource_tags`, `roots`+Today (10): `account_parents`, `accounts`, `delegated_administrators`, `delegated_services`, `organization`, `organizational_units`, `policies`, `policy_targets`, `resource_tags`, `roots`Diffed against: <https://raw.githubusercontent.com/boto/botocore/develop/botocore/data/organizations/2016-11-28/service-2.json>-- [ ] `Account parents (ListParents, POST /)` — connects synced accounts to organizational units and roots for cost allocation by organizational structure (high)-- [ ] `Policy targets (ListTargetsForPolicy, POST /)` — connects synced policies to accounts, organizational units, and roots for governance analysis (high)-- [ ] `Delegated administrators (ListDelegatedAdministrators, POST /)` — identifies accounts responsible for administering organization services (medium)-- [ ] `Delegated services for account (ListDelegatedServicesForAccount, POST /)` — connects delegated administrator accounts to the services they manage (medium)+- [x] `Account parents (ListParents, POST /)` — connects synced accounts to organizational units and roots for cost allocation by organizational structure (high)+- [x] `Policy targets (ListTargetsForPolicy, POST /)` — connects synced policies to accounts, organizational units, and roots for governance analysis (high)+- [x] `Delegated administrators (ListDelegatedAdministrators, POST /)` — identifies accounts responsible for administering organization services (medium)+- [x] `Delegated services for account (ListDelegatedServicesForAccount, POST /)` — connects delegated administrator accounts to the services they manage (medium)Note: Compared the official botocore operation model with the static endpoint map and extraction code. Organizational units already include parent IDs; account rows do not include their parent organizational unit. Account parents, policy targets, and delegated services require traversal; JSON RPC reads use POST /. Tags are already synced; policy content, service setup, and account creation plumbing were excluded.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/aws_organizations.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/aws_organizations.pyindex 7f4482598f2b..7f6e8dd87c4b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/aws_organizations.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/aws_organizations.py@@ -341,13 +341,12 @@ def _list_rows()-def _policy_rows(- client: AwsOrganizationsClient,- endpoint_config: AwsOrganizationsEndpointConfig,+def _per_policy_type(resumable_source_manager: ResumableSourceManager[AwsOrganizationsResumeConfig],logger: FilteringBoundLogger,+ rows_for_policy_type: Callable[[str, Optional[str]], Iterator[list[dict[str, Any]]]],) -> Iterator[list[dict[str, Any]]]:- """One walk per policy type, since `Filter` is required and takes a single type."""+ """One walk per policy type, since `ListPolicies` requires `Filter` and takes a single type."""filters = list(POLICY_FILTERS)start_token: Optional[str] = None@@ -364,6 +363,24 @@ def _policy_rows(logger.debug(f"Resuming policies from a saved policy type. policy_type={resumed_filter}")for policy_filter in filters:+ try:+ yield from rows_for_policy_type(policy_filter, start_token)+ except AwsOrganizationsError as error:+ if is_policy_type_unavailable(error):+ logger.debug(f"Skipping unavailable policy type. policy_type={policy_filter}")+ start_token = None+ continue+ raise+ start_token = None+++def _policy_rows(+ client: AwsOrganizationsClient,+ endpoint_config: AwsOrganizationsEndpointConfig,+ resumable_source_manager: ResumableSourceManager[AwsOrganizationsResumeConfig],+ logger: FilteringBoundLogger,+) -> Iterator[list[dict[str, Any]]]:+ def rows_for_policy_type(policy_filter: str, start_token: Optional[str]) -> Iterator[list[dict[str, Any]]]:pages = iter_pages(client,endpoint_config.operation,@@ -372,20 +389,187 @@ def _policy_rows(logger,start_token=start_token,)- try:- yield from _checkpointed_rows(- pages,- lambda items: [normalize_row(endpoint_config, item) for item in items],- resumable_source_manager,- work_key=policy_filter,+ return _checkpointed_rows(+ pages,+ lambda items: [normalize_row(endpoint_config, item) for item in items],+ resumable_source_manager,+ work_key=policy_filter,+ )++ yield from _per_policy_type(resumable_source_manager, logger, rows_for_policy_type)+++def _fan_out_rows(+ parent_pages: Iterator[Page],+ child_pages: Callable[[dict[str, Any]], Iterator[list[dict[str, Any]]]],+ resumable_source_manager: ResumableSourceManager[AwsOrganizationsResumeConfig],+ work_key: Optional[str] = None,+) -> Iterator[list[dict[str, Any]]]:+ """Child rows for every parent, resumable on the parent walk's page token.++ A parent page's children are yielded as one batch, so the cursor never sits mid-page and a+ resumed full refresh can't append the same children twice. The buffered rows are outside the+ staged cursor, so a safe point while they are held only re-reads them.+ """+ for parent_page in parent_pages:+ rows: list[dict[str, Any]] = []+ for parent in parent_page.items:+ for child_rows in child_pages(parent):+ rows.extend(child_rows)+ resumable_source_manager.safe_point()+ resumable_source_manager.save_state(+ AwsOrganizationsResumeConfig(next_token=parent_page.next_token, work_key=work_key)+ )+ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()+++def _child_pages(+ client: AwsOrganizationsClient,+ endpoint_config: AwsOrganizationsEndpointConfig,+ payload: dict[str, Any],+ parent_columns: dict[str, Any],+ skip_codes: frozenset[str],+ logger: FilteringBoundLogger,+) -> Iterator[list[dict[str, Any]]]:+ try:+ for child_page in iter_pages(+ client,+ endpoint_config.operation,+ {**payload, "MaxResults": endpoint_config.page_size},+ result_key(endpoint_config),+ logger,+ ):+ yield [{**normalize_row(endpoint_config, item), **parent_columns} for item in child_page.items]+ except AwsOrganizationsError as error:+ if error.code in skip_codes:+ logger.debug(f"Skipping parent removed mid-sync. endpoint={endpoint_config.name} parent={parent_columns}")+ return+ raise+++def _single_walk_pages(+ client: AwsOrganizationsClient,+ operation: str,+ items_key: str,+ resumable_source_manager: ResumableSourceManager[AwsOrganizationsResumeConfig],+ logger: FilteringBoundLogger,+) -> Iterator[Page]:+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ # Every saved state names the page after the one it covers, so a missing token means the+ # previous attempt finished the walk and only stopped before clearing its state.+ if resume is not None and not resume.next_token:+ logger.debug(f"Saved state marks the walk complete. operation={operation}")+ return iter(())+ start_token = resume.next_token if resume is not None else None+ return iter_pages(client, operation, {"MaxResults": MAX_RESULTS}, items_key, logger, start_token=start_token)+++def _account_parent_rows(+ client: AwsOrganizationsClient,+ endpoint_config: AwsOrganizationsEndpointConfig,+ resumable_source_manager: ResumableSourceManager[AwsOrganizationsResumeConfig],+ logger: FilteringBoundLogger,+) -> Iterator[list[dict[str, Any]]]:+ def parents_of(account: dict[str, Any]) -> Iterator[list[dict[str, Any]]]:+ account_id = account.get("Id")+ if not isinstance(account_id, str) or not account_id:+ return+ for rows in _child_pages(+ client,+ endpoint_config,+ {"ChildId": account_id},+ {"account_id": account_id},+ frozenset({"ChildNotFoundException"}),+ logger,+ ):+ yield [+ {"account_id": row["account_id"], "parent_id": row.get("id"), "parent_type": row.get("type")}+ for row in rows+ ]++ accounts = _single_walk_pages(client, "ListAccounts", "Accounts", resumable_source_manager, logger)+ yield from _fan_out_rows(accounts, parents_of, resumable_source_manager)+++def _policy_target_rows(+ client: AwsOrganizationsClient,+ endpoint_config: AwsOrganizationsEndpointConfig,+ resumable_source_manager: ResumableSourceManager[AwsOrganizationsResumeConfig],+ logger: FilteringBoundLogger,+) -> Iterator[list[dict[str, Any]]]:+ def rows_for_policy_type(policy_filter: str, start_token: Optional[str]) -> Iterator[list[dict[str, Any]]]:+ def targets_of(policy: dict[str, Any]) -> Iterator[list[dict[str, Any]]]:+ policy_id = policy.get("Id")+ if not isinstance(policy_id, str) or not policy_id:+ return iter(())+ return _child_pages(+ client,+ endpoint_config,+ {"PolicyId": policy_id},+ {"policy_id": policy_id, "policy_type": policy_filter},+ frozenset({"PolicyNotFoundException"}),+ logger,)- except AwsOrganizationsError as error:- if is_policy_type_unavailable(error):- logger.debug(f"Skipping unavailable policy type. policy_type={policy_filter}")- start_token = None- continue- raise- start_token = None++ policies = iter_pages(+ client,+ "ListPolicies",+ {"Filter": policy_filter, "MaxResults": MAX_RESULTS},+ "Policies",+ logger,+ start_token=start_token,+ )+ return _fan_out_rows(policies, targets_of, resumable_source_manager, work_key=policy_filter)++ yield from _per_policy_type(resumable_source_manager, logger, rows_for_policy_type)+++def _delegated_service_rows(+ client: AwsOrganizationsClient,+ endpoint_config: AwsOrganizationsEndpointConfig,+ resumable_source_manager: ResumableSourceManager[AwsOrganizationsResumeConfig],+ logger: FilteringBoundLogger,+) -> Iterator[list[dict[str, Any]]]:+ def services_of(administrator: dict[str, Any]) -> Iterator[list[dict[str, Any]]]:+ account_id = administrator.get("Id")+ if not isinstance(account_id, str) or not account_id:+ return iter(())+ return _child_pages(+ client,+ endpoint_config,+ {"AccountId": account_id},+ {"account_id": account_id},+ # The account was deregistered or left the organization after it was listed.+ frozenset({"AccountNotRegisteredException", "AccountNotFoundException"}),+ logger,+ )++ administrators = _single_walk_pages(+ client, "ListDelegatedAdministrators", "DelegatedAdministrators", resumable_source_manager, logger+ )+ yield from _fan_out_rows(administrators, services_of, resumable_source_manager)+++WALKS_BY_ENDPOINT: dict[+ str,+ Callable[+ [+ AwsOrganizationsClient,+ AwsOrganizationsEndpointConfig,+ ResumableSourceManager[AwsOrganizationsResumeConfig],+ FilteringBoundLogger,+ ],+ Iterator[list[dict[str, Any]]],+ ],+] = {+ "account_parents": _account_parent_rows,+ "delegated_services": _delegated_service_rows,+ "policies": _policy_rows,+ "policy_targets": _policy_target_rows,+}# Named fields instead of a 2/3-tuple return: semgrep's tuple-return-prefer-dataclass@@ -546,8 +730,9 @@ def get_rows(yield from _resource_tag_rows(client, endpoint_config, logger)return- if endpoint == "policies":- yield from _policy_rows(client, endpoint_config, resumable_source_manager, logger)+ resumable_rows = WALKS_BY_ENDPOINT.get(endpoint)+ if resumable_rows is not None:+ yield from resumable_rows(client, endpoint_config, resumable_source_manager, logger)else:yield from _list_rows(client, endpoint_config, resumable_source_manager, logger)@@ -566,6 +751,31 @@ def _permission_reason(error: AwsOrganizationsError) -> Optional[str]:return None+def _probe_first_child(+ client: AwsOrganizationsClient,+ parent_operation: str,+ parent_payload: dict[str, Any],+ parent_key: str,+ child_operation: str,+ child_param: str,+) -> None:+ next_token: Optional[str] = None+ while True:+ payload = {**parent_payload, "MaxResults": 1}+ if next_token:+ payload["NextToken"] = next_token+ parents = client.request(parent_operation, payload)+ for parent in parents.get(parent_key) or []:+ parent_id = parent.get("Id") if isinstance(parent, dict) else None+ if isinstance(parent_id, str):+ client.request(child_operation, {child_param: parent_id, "MaxResults": 1})+ return+ # AWS can return an empty page while more parents remain.+ next_token = parents.get("NextToken") or None+ if not next_token:+ return++def endpoint_permission_reason(client: AwsOrganizationsClient, endpoint: str) -> Optional[str]:"""Probe the calls a sync of this endpoint issues. `None` when reachable.@@ -588,6 +798,26 @@ def endpoint_permission_reason(client: AwsOrganizationsClient, endpoint: str) ->breakelif endpoint == "policies":client.request("ListPolicies", {"Filter": POLICY_FILTERS[0], "MaxResults": 1})+ elif endpoint == "account_parents":+ _probe_first_child(client, "ListAccounts", {}, "Accounts", "ListParents", "ChildId")+ elif endpoint == "policy_targets":+ _probe_first_child(+ client,+ "ListPolicies",+ {"Filter": POLICY_FILTERS[0]},+ "Policies",+ "ListTargetsForPolicy",+ "PolicyId",+ )+ elif endpoint == "delegated_services":+ _probe_first_child(+ client,+ "ListDelegatedAdministrators",+ {},+ "DelegatedAdministrators",+ "ListDelegatedServicesForAccount",+ "AccountId",+ )elif endpoint == "resource_tags":accounts = client.request("ListAccounts", {"MaxResults": 1})for account in accounts.get("Accounts") or []:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/canonical_descriptions.pyindex c7b362fde307..6670b3a690f6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/canonical_descriptions.py@@ -3,6 +3,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.can)CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {+ "account_parents": {+ "description": "The root or organizational unit that each account sits directly under. Join to accounts on account_id and to organizational_units or roots on parent_id.",+ "docs_url": "https://docs.aws.amazon.com/organizations/latest/APIReference/API_ListParents.html",+ "columns": {+ "account_id": "The unique identifier (ID) of the account, a 12-digit number.",+ "parent_id": "The unique identifier (ID) of the parent entity: a root ID starting with 'r-' or an organizational unit ID starting with 'ou-'.",+ "parent_type": "The type of the parent entity: ROOT or ORGANIZATIONAL_UNIT.",+ },+ },"accounts": {"description": "Every AWS account in the organization. The dimension table that maps a 12-digit account ID to a human-readable account name.","docs_url": "https://docs.aws.amazon.com/organizations/latest/APIReference/API_Account.html",@@ -17,6 +26,30 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"paths": "The paths of the account in the organization hierarchy, from the root down to the account.",},},+ "delegated_administrators": {+ "description": "Accounts in the organization that are designated as delegated administrators for an AWS service.",+ "docs_url": "https://docs.aws.amazon.com/organizations/latest/APIReference/API_DelegatedAdministrator.html",+ "columns": {+ "id": "The unique identifier (ID) of the delegated administrator's account.",+ "arn": "The Amazon Resource Name (ARN) of the delegated administrator's account.",+ "email": "The email address that is associated with the delegated administrator's AWS account.",+ "name": "The friendly name of the delegated administrator's account.",+ "status": "The status of the delegated administrator's account in the organization.",+ "state": "The lifecycle state of the delegated administrator's account in the organization.",+ "joined_method": "The method by which the delegated administrator's account joined the organization.",+ "joined_timestamp": "The date when the delegated administrator's account became a part of the organization.",+ "delegation_enabled_date": "The date when the account was made a delegated administrator.",+ },+ },+ "delegated_services": {+ "description": "The AWS services for which each delegated administrator account is a delegated administrator. One row per account and service.",+ "docs_url": "https://docs.aws.amazon.com/organizations/latest/APIReference/API_DelegatedService.html",+ "columns": {+ "account_id": "The unique identifier (ID) of the delegated administrator's account.",+ "service_principal": "The name of the AWS service, typically in the form servicename.amazonaws.com.",+ "delegation_enabled_date": "The date that the account became a delegated administrator for this service.",+ },+ },"organization": {"description": "The organization itself, including its management account and which functionality is enabled. One row.","docs_url": "https://docs.aws.amazon.com/organizations/latest/APIReference/API_Organization.html",@@ -53,6 +86,18 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"aws_managed": "True when the policy is managed by AWS, which means it can be attached to entities but not edited.",},},+ "policy_targets": {+ "description": "The roots, organizational units, and accounts that each policy is attached to. One row per attachment.",+ "docs_url": "https://docs.aws.amazon.com/organizations/latest/APIReference/API_PolicyTargetSummary.html",+ "columns": {+ "policy_id": "The unique identifier (ID) of the attached policy.",+ "policy_type": "The type of the attached policy, for example SERVICE_CONTROL_POLICY or TAG_POLICY.",+ "target_id": "The unique identifier (ID) of the policy target: a root, organizational unit, or account ID.",+ "arn": "The Amazon Resource Name (ARN) of the policy target.",+ "name": "The friendly name of the policy target.",+ "type": "The type of the policy target: ROOT, ORGANIZATIONAL_UNIT, or ACCOUNT.",+ },+ },"resource_tags": {"description": "Tags attached to accounts, roots, organizational units, and policies. One row per tag.","docs_url": "https://docs.aws.amazon.com/organizations/latest/APIReference/API_ListTagsForResource.html",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/settings.pyindex 4f96c9b60935..d96d458eda40 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/settings.py@@ -89,6 +89,14 @@ class AwsOrganizationsEndpointConfig:AWS_ORGANIZATIONS_ENDPOINTS: dict[str, AwsOrganizationsEndpointConfig] = {+ "account_parents": AwsOrganizationsEndpointConfig(+ name="account_parents",+ operation="ListParents",+ primary_key=["account_id", "parent_id"],+ result_key="Parents",+ page_size=MAX_RESULTS,+ iam_actions=("organizations:ListAccounts", "organizations:ListParents"),+ ),"accounts": AwsOrganizationsEndpointConfig(name="accounts",operation="ListAccounts",@@ -101,6 +109,27 @@ AWS_ORGANIZATIONS_ENDPOINTS: dict[str, AwsOrganizationsEndpointConfig] = {drop_keys=frozenset({"Status"}),iam_actions=("organizations:ListAccounts",),),+ "delegated_administrators": AwsOrganizationsEndpointConfig(+ name="delegated_administrators",+ operation="ListDelegatedAdministrators",+ primary_key=["id"],+ result_key="DelegatedAdministrators",+ page_size=MAX_RESULTS,+ timestamp_columns=("joined_timestamp", "delegation_enabled_date"),+ iam_actions=("organizations:ListDelegatedAdministrators",),+ ),+ "delegated_services": AwsOrganizationsEndpointConfig(+ name="delegated_services",+ operation="ListDelegatedServicesForAccount",+ primary_key=["account_id", "service_principal"],+ result_key="DelegatedServices",+ page_size=MAX_RESULTS,+ timestamp_columns=("delegation_enabled_date",),+ iam_actions=(+ "organizations:ListDelegatedAdministrators",+ "organizations:ListDelegatedServicesForAccount",+ ),+ ),"organization": AwsOrganizationsEndpointConfig(name="organization",operation="DescribeOrganization",@@ -127,6 +156,14 @@ AWS_ORGANIZATIONS_ENDPOINTS: dict[str, AwsOrganizationsEndpointConfig] = {page_size=MAX_RESULTS,iam_actions=("organizations:ListPolicies",),),+ "policy_targets": AwsOrganizationsEndpointConfig(+ name="policy_targets",+ operation="ListTargetsForPolicy",+ primary_key=["policy_id", "target_id"],+ result_key="Targets",+ page_size=MAX_RESULTS,+ iam_actions=("organizations:ListPolicies", "organizations:ListTargetsForPolicy"),+ ),"resource_tags": AwsOrganizationsEndpointConfig(name="resource_tags",operation="ListTagsForResource",@@ -158,10 +195,14 @@ ENDPOINTS = tuple(AWS_ORGANIZATIONS_ENDPOINTS.keys())INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}ENDPOINT_DESCRIPTIONS: dict[str, str] = {+ "account_parents": "The root or organizational unit each account sits directly under. One row per account.","accounts": "Every AWS account in the organization, with its name, email, state, and the time it joined.",+ "delegated_administrators": "Accounts registered as delegated administrators for an AWS service in the organization.",+ "delegated_services": "The AWS services each delegated administrator account administers. One row per account and service.","organization": "The organization itself: its ID, feature set, and management account. One row per sync.","organizational_units": "Organizational units in the account hierarchy, each with the parent it sits under.","policies": "Policies in the organization, one row per policy, across every policy type the organization supports.",+ "policy_targets": "The roots, organizational units, and accounts each policy is attached to. One row per attachment.","resource_tags": "Tags attached to accounts, roots, organizational units, and policies. One row per tag.","roots": "Roots of the organization, with the policy types enabled on each.",}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/source.pyindex 828c00cef7cc..b289d227f89a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/source.py@@ -68,7 +68,7 @@ class AwsOrganizationsSource(ResumableSource[AwsOrganizationsSourceConfig, AwsOr"AWS Organizations request failed: SignatureDoesNotMatch": "AWS rejected the request signature. Please re-enter the secret access key.","AWS Organizations request failed: InvalidSignatureException": "AWS rejected the request signature. If you are using temporary credentials, the session token has expired.","AWS Organizations request failed: ExpiredTokenException": "The AWS session token has expired. Please reconnect with fresh credentials.",- "AWS Organizations request failed: AccessDeniedException": f"These AWS credentials are missing AWS Organizations read permissions. Grant organizations:DescribeOrganization, organizations:ListAccounts, organizations:ListRoots, organizations:ListOrganizationalUnitsForParent, organizations:ListPolicies and organizations:ListTagsForResource to the IAM user or role. {_MANAGEMENT_ACCOUNT_HINT}",+ "AWS Organizations request failed: AccessDeniedException": f"These AWS credentials are missing AWS Organizations read permissions. Grant organizations:DescribeOrganization, organizations:ListAccounts, organizations:ListParents, organizations:ListRoots, organizations:ListOrganizationalUnitsForParent, organizations:ListPolicies, organizations:ListTargetsForPolicy, organizations:ListTagsForResource, organizations:ListDelegatedAdministrators and organizations:ListDelegatedServicesForAccount to the IAM user or role. {_MANAGEMENT_ACCOUNT_HINT}","AWS Organizations request failed: AWSOrganizationsNotInUseException": "This AWS account isn't a member of an organization. Connect a key from an account in an AWS organization.","AWS access key ID and secret access key are required": "Enter both an AWS access key ID and a secret access key.",}@@ -150,7 +150,7 @@ class AwsOrganizationsSource(ResumableSource[AwsOrganizationsSourceConfig, AwsOrlabel="AWS Organizations",caption="""Sync your AWS organization structure into the PostHog Data warehouse, so cost and usage data can be read by account name, organizational unit, and tag.-Create an IAM user or role with the `organizations:DescribeOrganization`, `organizations:ListAccounts`, `organizations:ListRoots`, `organizations:ListOrganizationalUnitsForParent`, `organizations:ListPolicies` and `organizations:ListTagsForResource` permissions, then paste its access key ID and secret access key. Add a session token too if you are using temporary credentials.+Create an IAM user or role with the `organizations:DescribeOrganization`, `organizations:ListAccounts`, `organizations:ListParents`, `organizations:ListRoots`, `organizations:ListOrganizationalUnitsForParent`, `organizations:ListPolicies`, `organizations:ListTargetsForPolicy`, `organizations:ListTagsForResource`, `organizations:ListDelegatedAdministrators` and `organizations:ListDelegatedServicesForAccount` permissions, then paste its access key ID and secret access key. Add a session token too if you are using temporary credentials.Use a key from the management account or from a member account that is a delegated administrator. Keys from other member accounts can only read the organization table.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/tests/test_aws_organizations.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/tests/test_aws_organizations.pyindex bc1e7d438bb6..d07612cc5c29 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/tests/test_aws_organizations.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/tests/test_aws_organizations.py@@ -284,6 +284,112 @@ class TestResourceTags:]+class TestFanOut:+ @pytest.mark.parametrize(+ "endpoint,parent_operation,parent_key,child_operation,child_param,child_key,child_item,missing_code,parent_column",+ [+ (+ "account_parents",+ "ListAccounts",+ "Accounts",+ "ListParents",+ "ChildId",+ "Parents",+ {"Id": "ou-exam-aaaaaaaa", "Type": "ORGANIZATIONAL_UNIT"},+ "ChildNotFoundException",+ "account_id",+ ),+ (+ "policy_targets",+ "ListPolicies",+ "Policies",+ "ListTargetsForPolicy",+ "PolicyId",+ "Targets",+ {"TargetId": "111111111111", "Type": "ACCOUNT", "Name": "prod"},+ "PolicyNotFoundException",+ "policy_id",+ ),+ (+ "delegated_services",+ "ListDelegatedAdministrators",+ "DelegatedAdministrators",+ "ListDelegatedServicesForAccount",+ "AccountId",+ "DelegatedServices",+ {"ServicePrincipal": "guardduty.amazonaws.com", "DelegationEnabledDate": JAN_2025},+ "AccountNotRegisteredException",+ "account_id",+ ),+ ],+ )+ def test_every_child_page_is_kept_with_its_parent_and_a_parent_removed_mid_sync_is_skipped(+ self,+ endpoint: str,+ parent_operation: str,+ parent_key: str,+ child_operation: str,+ child_param: str,+ child_key: str,+ child_item: dict[str, Any],+ missing_code: str,+ parent_column: str,+ ) -> None:+ def responder(request: RecordedRequest) -> ScriptedResponse:+ payload = request.json()+ operation = request_operation(request)+ if operation == parent_operation:+ if endpoint == "policy_targets" and payload["Filter"] != "SERVICE_CONTROL_POLICY":+ return ScriptedResponse(json=page(parent_key, []))+ if payload.get("NextToken") is None:+ return ScriptedResponse(json=page(parent_key, [{"Id": "parent-a"}], next_token="parents-2"))+ return ScriptedResponse(json=page(parent_key, [{"Id": "parent-gone"}, {"Id": "parent-b"}]))+ assert operation == child_operation+ if payload[child_param] == "parent-gone":+ return aws_error(missing_code, "The entity no longer exists.")+ if payload[child_param] == "parent-a" and payload.get("NextToken") is None:+ return ScriptedResponse(json=page(child_key, [child_item], next_token="children-2"))+ return ScriptedResponse(json=page(child_key, [child_item]))++ result = driver().run(endpoint, responder)++ assert result.raised is None+ assert [[row[parent_column] for row in item] for item in result.items] == [+ ["parent-a", "parent-a"],+ ["parent-b"],+ ]+ child_payloads = [+ request.json() for request in result.requests if request_operation(request) == child_operation+ ]+ assert all(payload["MaxResults"] == MAX_RESULTS for payload in child_payloads)++ def test_account_parents_rows_name_the_parent_and_resume_the_account_walk(self) -> None:+ def responder(request: RecordedRequest) -> ScriptedResponse:+ if request_operation(request) == "ListAccounts":+ return ScriptedResponse(json=page("Accounts", [{"Id": "222222222222"}]))+ return ScriptedResponse(json=page("Parents", [{"Id": "r-exam", "Type": "ROOT"}]))++ result = driver().run(+ "account_parents", responder, resume_state=AwsOrganizationsResumeConfig(next_token="accounts-2")+ )++ assert result.raised is None+ assert result.requests[0].json() == {"MaxResults": MAX_RESULTS, "NextToken": "accounts-2"}+ assert result.rows == [{"account_id": "222222222222", "parent_id": "r-exam", "parent_type": "ROOT"}]++ @pytest.mark.parametrize("endpoint", ["account_parents", "delegated_services"])+ def test_a_walk_saved_as_complete_is_not_read_again(self, endpoint: str) -> None:+ result = driver().run(+ endpoint,+ always(aws_error("ServiceException", "unexpected request")),+ resume_state=AwsOrganizationsResumeConfig(next_token=None),+ )++ assert result.raised is None+ assert result.requests == []+ assert result.rows == []++class TestResume:def test_an_expired_saved_token_restarts_the_walk_instead_of_failing(self) -> None:def responder(request: RecordedRequest) -> ScriptedResponse:@@ -310,9 +416,10 @@ class TestResume:assert result.raised is Noneassert [request.json()["Filter"] for request in result.requests] == list(POLICY_FILTERS[1:])- def test_policies_resume_continues_a_half_walked_policy_type(self) -> None:+ @pytest.mark.parametrize("endpoint", ["policies", "policy_targets"])+ def test_policies_resume_continues_a_half_walked_policy_type(self, endpoint: str) -> None:result = driver().run(- "policies",+ endpoint,always(ScriptedResponse(json=page("Policies", [{"Id": "p-example"}]))),resume_state=AwsOrganizationsResumeConfig(next_token="page-two", work_key=POLICY_FILTERS[1]),)@@ -322,7 +429,9 @@ class TestResume:"ListPolicies",{"Filter": POLICY_FILTERS[1], "MaxResults": MAX_RESULTS, "NextToken": "page-two"},)- assert [request.json()["Filter"] for request in result.requests] == list(POLICY_FILTERS[1:])+ assert [+ request.json()["Filter"] for request in result.requests if request_operation(request) == "ListPolicies"+ ] == list(POLICY_FILTERS[1:])class TestValidateCredentials:@@ -384,7 +493,11 @@ class TestProbeEndpointPermissions:match request_operation(request):case "ListPolicies":return aws_error("AccessDeniedException", "not authorized to perform organizations:ListPolicies")+ case "ListParents":+ return aws_error("AccessDeniedException", "not authorized to perform organizations:ListParents")case "ListAccounts":+ if request.json().get("NextToken") is None:+ return ScriptedResponse(json=page("Accounts", [], next_token="accounts-2"))return ScriptedResponse(json=page("Accounts", [{"Id": "111111111111"}]))case "ListRoots":return ScriptedResponse(json=page("Roots", [{"Id": "r-exam"}]))@@ -395,10 +508,15 @@ class TestProbeEndpointPermissions:with scripted_network(responder) as network:reasons = probe_endpoint_permissions(- "AKIAEXAMPLE", "secret", None, ["accounts", "organization", "organizational_units", "policies"]+ "AKIAEXAMPLE",+ "secret",+ None,+ ["accounts", "organization", "organizational_units", "policies", "account_parents"],)assert reasons["accounts"] is None+ assert reasons["account_parents"] is not None+ assert "organizations:ListParents" in reasons["account_parents"]assert reasons["organization"] is Noneassert reasons["organizational_units"] is Noneassert reasons["policies"] is not None@@ -409,6 +527,9 @@ class TestProbeEndpointPermissions:"ListRoots","ListOrganizationalUnitsForParent","ListPolicies",+ "ListAccounts",+ "ListAccounts",+ "ListParents",]@pytest.mark.parametrize("error", [AwsOrganizationsRetryableError("TooManyRequestsException", "slow down")])