Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

22.0 KB
Raw Blame
"""Tests for regen_partitions.

Run with: pytest tools/aws -q  (or: make aws-data-test)
"""
from __future__ import annotations

import gzip
import json
from pathlib import Path

import pytest

import regen_partitions as r


ENTITIES = """
export const AWS_REGIONS_AND_RULES: readonly (string | symbol)[] = [
  'us-east-1', // US East (N. Virginia)
  'us-gov-west-1', // AWS GovCloud (US-West)
  RULE_S3_WEBSITE_REGIONAL_SUBDOMAIN,
  'cn-north-1', // China (Beijing)
  'eu-north-1', // Europe (Stockholm)
  RULE_CLASSIC_PARTITION_BECOMES_OPT_IN,
  'ap-east-1', // Asia Pacific (Hong Kong)
  'cn-northwest-1', // China (Ningxia)
];

export const AWS_REGIONS = AWS_REGIONS_AND_RULES.filter((x) => typeof x === 'string').sort();
"""

PARTITIONS = {
    "partitions": [
        {
            "id": "aws-cn",
            "outputs": {"dnsSuffix": "amazonaws.com.cn", "dualStackDnsSuffix": "api.amazonwebservices.com.cn",
                        "implicitGlobalRegion": "cn-northwest-1", "name": "aws-cn",
                        "supportsDualStack": True, "supportsFIPS": True},
            "regionRegex": "^cn\\-\\w+\\-\\d+$",
            "regions": {"aws-cn-global": {"description": "aws-cn global region"},
                        "cn-north-1": {"description": "China (Beijing)"},
                        "cn-northwest-1": {"description": "China (Ningxia)"}},
        },
        {
            "id": "aws",
            "outputs": {"dnsSuffix": "amazonaws.com", "dualStackDnsSuffix": "api.aws",
                        "implicitGlobalRegion": "us-east-1", "name": "aws",
                        "supportsDualStack": True, "supportsFIPS": True},
            "regionRegex": "^(us|eu|ap)\\-\\w+\\-\\d+$",
            "regions": {"aws-global": {"description": "aws global region"},
                        "us-east-1": {"description": "US East (N. Virginia)"},
                        "eu-north-1": {"description": "Europe (Stockholm)"}},
        },
        {
            "id": "aws-eusc",
            "outputs": {"dnsSuffix": "amazonaws.eu", "dualStackDnsSuffix": "api.amazonwebservices.eu",
                        "implicitGlobalRegion": "eusc-de-east-1", "name": "aws-eusc",
                        "supportsDualStack": True, "supportsFIPS": True},
            "regionRegex": "^eusc\\-(de)\\-\\w+\\-\\d+$",
            "regions": {"eusc-de-east-1": {"description": "EU (Germany)"}},
        },
    ],
    "version": "1.1",
}

ENDPOINTS = {
    "partitions": [
        {
            "partition": "aws",
            "partitionName": "AWS Standard",
            "dnsSuffix": "amazonaws.com",
            "defaults": {"hostname": "{service}.{region}.{dnsSuffix}"},
            "regions": {"us-east-1": {"description": "US East (N. Virginia)"},
                        "eu-north-1": {"description": "Europe (Stockholm)"},
                        "ap-east-1": {"description": "Asia Pacific (Hong Kong)"}},
            "services": {
                "iam": {"endpoints": {"aws-global": {"credentialScope": {"region": "us-east-1"},
                                                     "hostname": "iam.amazonaws.com"}},
                        "isRegionalized": False, "partitionEndpoint": "aws-global"},
                "sts": {"endpoints": {"aws-global": {"credentialScope": {"region": "us-east-1"},
                                                     "hostname": "sts.amazonaws.com"},
                                      "us-east-1": {}},
                        "partitionEndpoint": "aws-global"},
                "codecatalyst": {"endpoints": {"aws-global": {"hostname": "codecatalyst.global.api.aws"}},
                                 "isRegionalized": False, "partitionEndpoint": "aws-global"},
                "nohost": {"endpoints": {"aws-global": {"credentialScope": {"region": "us-east-1"}}},
                           "isRegionalized": False, "partitionEndpoint": "aws-global"},
                "sqs": {"endpoints": {"us-east-1": {}}},
                "s3": {"defaults": {"variants": [{"hostname": "{service}.dualstack.{region}.{dnsSuffix}", "tags": ["dualstack"]}]},
                       "endpoints": {"us-east-1": {}, "eu-north-1": {}, "aws-global": {"hostname": "s3.amazonaws.com"}}},
            },
        },
        {
            "partition": "aws-cn",
            "partitionName": "AWS China",
            "dnsSuffix": "amazonaws.com.cn",
            "defaults": {"hostname": "{service}.{region}.{dnsSuffix}"},
            "regions": {"cn-north-1": {"description": "China (Beijing)"},
                        "cn-northwest-1": {"description": "China (Ningxia)"}},
            "services": {
                "iam": {"endpoints": {"aws-cn-global": {"credentialScope": {"region": "cn-north-1"},
                                                        "hostname": "iam.cn-north-1.amazonaws.com.cn"}},
                        "isRegionalized": False, "partitionEndpoint": "aws-cn-global"},
                "sqs": {"endpoints": {"cn-north-1": {}}},
                "s3": {"endpoints": {"cn-north-1": {"variants": [{"hostname": "s3.dualstack.cn-north-1.amazonaws.com.cn", "tags": ["dualstack"]}]},
                                     "cn-northwest-1": {}}},
            },
        },
        {
            "partition": "aws-eusc",
            "partitionName": "AWS EUSC",
            "dnsSuffix": "amazonaws.eu",
            "defaults": {"hostname": "{service}.{region}.{dnsSuffix}"},
            "regions": {"eusc-de-east-1": {"description": "EU (Germany)"}},
            "services": {"sqs": {"endpoints": {"eusc-de-east-1": {}}}},
        },
    ],
    "version": 3,
}


def by_id(document: dict, partition_id: str) -> dict:
    return next(p for p in document["partitions"] if p["id"] == partition_id)


def region(partition: dict, region_id: str) -> dict:
    return next(x for x in partition["regions"] if x["id"] == region_id)


# --------------------------------------------------------------------------- #
# CDK rules
# --------------------------------------------------------------------------- #
def test_parse_cdk_entities_keeps_file_order_with_rule_markers():
    assert r.parse_cdk_entities(ENTITIES) == [
        "us-east-1", "us-gov-west-1", r.RULE_S3_WEBSITE, "cn-north-1", "eu-north-1",
        r.RULE_OPT_IN, "ap-east-1", "cn-northwest-1",
    ]


def test_parse_cdk_entities_requires_both_rules():
    with pytest.raises(ValueError, match="rule markers"):
        r.parse_cdk_entities("export const AWS_REGIONS_AND_RULES = [\n  'us-east-1',\n];\n")


@pytest.mark.parametrize("region_id,partition,opt_in,dash", [
    ("us-east-1", "aws", False, True),
    ("us-gov-west-1", "aws-us-gov", False, True),
    ("cn-north-1", "aws-cn", False, False),
    ("eu-north-1", "aws", False, False),
    ("ap-east-1", "aws", True, False),
    ("cn-northwest-1", "aws-cn", False, False),
    ("eu-nowhere-9", "aws", True, False),
    ("cn-nowhere-9", "aws-cn", False, False),
])
def test_cdk_flags_follow_the_two_rules(region_id, partition, opt_in, dash):
    entries = r.parse_cdk_entities(ENTITIES)
    assert r.cdk_flags(entries, region_id, partition) == (opt_in, dash)


# --------------------------------------------------------------------------- #
# Build
# --------------------------------------------------------------------------- #
def test_build_puts_the_commercial_partition_first_then_sorts_by_id():
    document = r.build(PARTITIONS, ENDPOINTS, r.parse_cdk_entities(ENTITIES), None, "test")
    assert [p["id"] for p in document["partitions"]] == ["aws", "aws-cn", "aws-eusc"]


def test_build_unions_regions_from_both_files_and_excludes_the_pseudo_region():
    document = r.build(PARTITIONS, ENDPOINTS, r.parse_cdk_entities(ENTITIES), None, "test")
    aws = by_id(document, "aws")
    assert [x["id"] for x in aws["regions"]] == ["ap-east-1", "eu-north-1", "us-east-1"]
    assert region(aws, "ap-east-1")["description"] == "Asia Pacific (Hong Kong)"
    assert aws["globalPseudoRegion"] == "aws-global"
    assert by_id(document, "aws-eusc")["globalPseudoRegion"] is None


def test_build_carries_partition_outputs_and_names():
    document = r.build(PARTITIONS, ENDPOINTS, r.parse_cdk_entities(ENTITIES), None, "test")
    cn = by_id(document, "aws-cn")
    assert cn["name"] == "AWS China"
    assert cn["dnsSuffix"] == "amazonaws.com.cn"
    assert cn["dualStackDnsSuffix"] == "api.amazonwebservices.com.cn"
    assert cn["implicitGlobalRegion"] == "cn-northwest-1"
    assert cn["regionRegex"] == "^cn\\-\\w+\\-\\d+$"
    assert cn["supportsDualStack"] is True and cn["supportsFips"] is True
    assert cn["services"] == ["iam", "s3", "sqs"]


def test_build_lists_the_regions_with_an_s3_dualstack_variant():
    document = r.build(PARTITIONS, ENDPOINTS, r.parse_cdk_entities(ENTITIES), None, "test")
    assert by_id(document, "aws")["s3DualStackRegions"] == ["eu-north-1", "us-east-1"]
    assert by_id(document, "aws-cn")["s3DualStackRegions"] == ["cn-north-1"]
    assert by_id(document, "aws-eusc")["s3DualStackRegions"] == []


def test_build_derives_global_services_from_partition_endpoint():
    document = r.build(PARTITIONS, ENDPOINTS, r.parse_cdk_entities(ENTITIES), None, "test")
    aws = by_id(document, "aws")["globalServices"]
    assert aws["iam"] == {"hostname": "iam.amazonaws.com", "signingRegion": "us-east-1", "regionalized": False}
    # isRegionalized absent means regional endpoints exist beside the global host (STS)
    assert aws["sts"] == {"hostname": "sts.amazonaws.com", "signingRegion": "us-east-1", "regionalized": True}
    # no credentialScope: the partition's implicit global region signs
    assert aws["codecatalyst"]["signingRegion"] == "us-east-1"
    # no hostname: the partition's default template with the endpoint key as region
    assert aws["nohost"]["hostname"] == "nohost.aws-global.amazonaws.com"
    assert "sqs" not in aws
    assert by_id(document, "aws-cn")["globalServices"]["iam"]["signingRegion"] == "cn-north-1"
    assert by_id(document, "aws-eusc")["globalServices"] == {}


def test_build_applies_opt_in_and_dash_form_flags():
    document = r.build(PARTITIONS, ENDPOINTS, r.parse_cdk_entities(ENTITIES), None, "test")
    aws = by_id(document, "aws")
    assert region(aws, "us-east-1") == {"id": "us-east-1", "description": "US East (N. Virginia)",
                                        "optIn": False, "s3WebsiteDashForm": True}
    assert region(aws, "ap-east-1")["optIn"] is True
    assert region(by_id(document, "aws-cn"), "cn-north-1")["optIn"] is False


def test_build_without_cdk_carries_flags_from_the_previous_file():
    with_cdk = r.build(PARTITIONS, ENDPOINTS, r.parse_cdk_entities(ENTITIES), None, "test")
    carried = r.build(PARTITIONS, ENDPOINTS, None, with_cdk, "test")
    assert r.strip_source(r.render(carried)) == r.strip_source(r.render(with_cdk))
    assert "carried over" in carried["_source"]["regionFlags"]


def test_build_without_cdk_refuses_a_region_it_cannot_carry():
    with pytest.raises(ValueError, match="no aws-cdk checkout"):
        r.build(PARTITIONS, ENDPOINTS, None, {"partitions": []}, "test")


def test_build_emits_the_signing_names_it_is_given_or_carries_the_previous_ones():
    fresh = r.build(PARTITIONS, ENDPOINTS, r.parse_cdk_entities(ENTITIES), None, "test",
                    {"ecr": ["api.ecr"]})
    assert fresh["signingNames"] == {"ecr": ["api.ecr"]}
    carried = r.build(PARTITIONS, ENDPOINTS, r.parse_cdk_entities(ENTITIES), fresh, "test")
    assert carried["signingNames"] == {"ecr": ["api.ecr"]}
    assert r.build(PARTITIONS, ENDPOINTS, r.parse_cdk_entities(ENTITIES), None, "test")["signingNames"] == {}


def test_collect_signing_names_keeps_only_the_services_whose_signing_name_differs(tmp_path):
    def model(service, version, endpoint_prefix, signing_name=None):
        metadata = {"endpointPrefix": endpoint_prefix}
        if signing_name:
            metadata["signingName"] = signing_name
        target = tmp_path / service / version / "service-2.json"
        target.parent.mkdir(parents=True)
        target.write_text(json.dumps({"metadata": metadata}))

    model("ecr", "2015-09-21", "api.ecr", "ecr")
    model("bedrock-runtime", "2023-09-30", "bedrock-runtime", "bedrock")
    model("bedrock-agent", "2023-06-05", "bedrock-agent", "bedrock")
    model("sqs", "2012-11-05", "sqs", "sqs")
    model("ssm", "2014-11-06", "ssm")
    model("old-ecr", "2014-01-01", "api.ecr", "legacy")
    model("old-ecr", "2015-09-21", "api.ecr", "ecr")
    (tmp_path / "partitions.json").write_text("{}")

    assert r.collect_signing_names(tmp_path) == {"bedrock": ["bedrock-agent", "bedrock-runtime"], "ecr": ["api.ecr"]}


def test_collect_signing_names_reads_the_gzipped_models_the_botocore_wheel_ships(tmp_path):
    # The pip package ships service-2.json.gz; the checkout ships it plain. Both must agree, or
    # aws-data-check fails in CI, which resolves botocore to the installed package.
    target = tmp_path / "ecr" / "2015-09-21" / "service-2.json.gz"
    target.parent.mkdir(parents=True)
    with gzip.open(target, "wt", encoding="utf-8") as handle:
        json.dump({"metadata": {"endpointPrefix": "api.ecr", "signingName": "ecr"}}, handle)

    assert r.collect_signing_names(tmp_path) == {"ecr": ["api.ecr"]}


def partition_is(partition_id):
    return {"fn": "stringEquals",
            "argv": [{"fn": "getAttr", "argv": [{"ref": "PartitionResult"}, "name"]}, partition_id]}


def endpoint_rule(conditions, url="https://example"):
    return {"type": "endpoint", "conditions": conditions, "endpoint": {"url": url}}


def test_ruleset_partitions_counts_only_explicit_branches_that_resolve_to_an_endpoint():
    rules = [
        {"type": "tree", "conditions": [partition_is("aws-eusc")],
         "rules": [endpoint_rule([], "https://iam.eusc-de-east-1.amazonaws.eu")]},
        endpoint_rule([partition_is("aws-iso-e")]),
        {"type": "error", "conditions": [partition_is("aws-iso-f")], "error": "not here"},
        {"type": "tree", "conditions": [partition_is("aws-iso-b")],
         "rules": [{"type": "error", "conditions": [], "error": "FIPS is not supported"}]},
        endpoint_rule([{"fn": "not", "argv": [partition_is("aws-cn")]}]),
        endpoint_rule([], "https://iam.{Region}.{PartitionResult#dnsSuffix}"),
    ]

    assert r.ruleset_partitions(rules) == {"aws-eusc", "aws-iso-e"}


def write_service(root, service, version, metadata, rules=None, gzipped=False):
    directory = root / service / version
    directory.mkdir(parents=True)
    (directory / "service-2.json").write_text(json.dumps({"metadata": metadata}))
    if rules is not None:
        document = {"version": "1.0", "parameters": {}, "rules": rules}
        if gzipped:
            with gzip.open(directory / "endpoint-rule-set-1.json.gz", "wt", encoding="utf-8") as handle:
                json.dump(document, handle)
        else:
            (directory / "endpoint-rule-set-1.json").write_text(json.dumps(document))


def test_collect_ruleset_partitions_keys_by_endpoint_prefix_and_reads_gzipped_rulesets(tmp_path):
    write_service(tmp_path, "iam", "2010-05-08", {"endpointPrefix": "iam"},
                  [endpoint_rule([partition_is("aws-eusc")])], gzipped=True)
    write_service(tmp_path, "sqs", "2012-11-05", {"endpointPrefix": "sqs"}, [endpoint_rule([])])
    write_service(tmp_path, "nomodel", "2020-01-01", {"endpointPrefix": "nomodel"})

    assert r.collect_ruleset_partitions(tmp_path) == {"iam": ["aws-eusc"]}


def test_build_adds_ruleset_partitions_only_for_services_endpoints_json_lists_somewhere():
    document = r.build(PARTITIONS, ENDPOINTS, r.parse_cdk_entities(ENTITIES), None, "test", {},
                       {"iam": ["aws-eusc"], "fis": ["aws-eusc"], "sqs": ["aws-eusc"]})
    eusc = by_id(document, "aws-eusc")

    assert eusc["services"] == ["iam", "sqs"]
    assert eusc["servicesFromRulesets"] == ["iam"]
    assert by_id(document, "aws")["servicesFromRulesets"] == []


def flag_is(parameter, value):
    return {"fn": "booleanEquals", "argv": [{"ref": parameter}, value]}


PLAIN = [flag_is("UseFIPS", False), flag_is("UseDualStack", False)]


def test_signin_hosts_reads_only_the_plain_data_plane_branches():
    rules = [
        {"type": "tree", "conditions": [flag_is("IsControlPlane", True)],
         "rules": [endpoint_rule([partition_is("aws")], "https://signin.{Region}.api.aws")]},
        endpoint_rule([flag_is("IsOAuthEndpoint", True), partition_is("aws")], "https://{Region}.oauth.signin.aws"),
        endpoint_rule(PLAIN + [partition_is("aws")], "https://{Region}.signin.aws.amazon.com"),
        endpoint_rule(PLAIN + [partition_is("aws-cn")], "https://{Region}.signin.amazonaws.cn"),
        endpoint_rule([flag_is("UseFIPS", True), flag_is("UseDualStack", False), partition_is("aws-us-gov")],
                      "https://{Region}.signin-fips.amazonaws-us-gov.com"),
        endpoint_rule(PLAIN, "https://{Region}.signin.{PartitionResult#dnsSuffix}"),
        {"type": "error", "conditions": PLAIN + [partition_is("aws-iso")], "error": "not here"},
        {"type": "tree", "conditions": [],
         "rules": [{"type": "tree", "conditions": [partition_is("aws-eusc")],
                    "rules": [endpoint_rule(PLAIN, "https://{Region}.signin.amazonaws-eusc.eu"),
                              endpoint_rule([partition_is("aws")] + PLAIN, "https://{Region}.signin.aws.amazon.com")]}]},
    ]

    assert r.signin_hosts(rules) == {
        "aws": "{region}.signin.aws.amazon.com",
        "aws-cn": "{region}.signin.amazonaws.cn",
        "aws-eusc": "{region}.signin.amazonaws-eusc.eu",
    }


def test_signin_hosts_refuses_two_hosts_for_one_partition():
    rules = [endpoint_rule(PLAIN + [partition_is("aws")], "https://{Region}.signin.aws.amazon.com"),
             endpoint_rule(PLAIN + [partition_is("aws")], "https://{Region}.signin.example.com")]

    with pytest.raises(ValueError, match="aws resolves to both"):
        r.signin_hosts(rules)


def test_collect_signin_hosts_reads_the_newest_gzipped_ruleset_or_nothing(tmp_path):
    assert r.collect_signin_hosts(tmp_path) == {}
    write_service(tmp_path, "signin", "2023-01-01", {"endpointPrefix": "signin"},
                  [endpoint_rule(PLAIN + [partition_is("aws-cn")], "https://{Region}.signin.amazonaws.cn")],
                  gzipped=True)

    assert r.collect_signin_hosts(tmp_path) == {"aws-cn": "{region}.signin.amazonaws.cn"}


def test_build_adds_the_signin_host_only_where_one_is_known():
    document = r.build(PARTITIONS, ENDPOINTS, r.parse_cdk_entities(ENTITIES), None, "test",
                       signin={"aws-cn": "{region}.signin.amazonaws.cn"})

    assert by_id(document, "aws-cn")["signinHost"] == "{region}.signin.amazonaws.cn"
    assert "signinHost" not in by_id(document, "aws")


def test_build_drops_a_signing_name_whose_own_prefix_service_is_listed_nowhere():
    signing_names = {"execute-api": ["participant.connect"], "ecr": ["api.ecr"], "iam": ["iam-toolbox"]}
    document = r.build(PARTITIONS, ENDPOINTS, r.parse_cdk_entities(ENTITIES), None, "test",
                       signing_names, None, {"execute-api", "iam", "sqs"})

    assert document["signingNames"] == {"ecr": ["api.ecr"], "iam": ["iam-toolbox"]}


def test_collect_self_signed_prefixes_lists_services_signing_with_their_own_prefix(tmp_path):
    write_service(tmp_path, "apigatewaymanagementapi", "2018-11-29", {"endpointPrefix": "execute-api"})
    write_service(tmp_path, "connectparticipant", "2018-09-07",
                  {"endpointPrefix": "participant.connect", "signingName": "execute-api"})

    assert r.collect_self_signed_prefixes(tmp_path) == {"execute-api"}


def test_build_refuses_a_partition_missing_from_endpoints_json():
    endpoints = {"partitions": [p for p in ENDPOINTS["partitions"] if p["partition"] != "aws-eusc"], "version": 3}
    with pytest.raises(ValueError, match="aws-eusc"):
        r.build(PARTITIONS, endpoints, r.parse_cdk_entities(ENTITIES), None, "test")


# --------------------------------------------------------------------------- #
# CLI
# --------------------------------------------------------------------------- #
def write_sources(tmp_path: Path) -> tuple[Path, Path]:
    data = tmp_path / "botocore-data"
    data.mkdir()
    (data / "partitions.json").write_text(json.dumps(PARTITIONS))
    (data / "endpoints.json").write_text(json.dumps(ENDPOINTS))
    cdk = tmp_path / "cdk"
    entities = cdk / r.CDK_ENTITIES
    entities.parent.mkdir(parents=True)
    entities.write_text(ENTITIES)
    return data, cdk


def test_cli_writes_then_reports_up_to_date_then_detects_drift(tmp_path, capsys):
    data, cdk = write_sources(tmp_path)
    output = tmp_path / "out" / "partitions.json"
    common = ["--botocore-data", str(data), "--cdk", str(cdk), "--output", str(output)]

    assert r.main(["--check"] + common) == 1
    assert "differs from a fresh generation" in capsys.readouterr().out

    assert r.main(common) == 0
    assert "3 partitions, 6 regions" in capsys.readouterr().out
    document = json.loads(output.read_text())
    assert document["_source"]["generator"] == "tools/aws/regen_partitions.py"
    assert document["_source"]["partitionsJsonVersion"] == "1.1"
    assert document["_source"]["endpointsJsonVersion"] == 3

    assert r.main(["--check"] + common) == 0
    assert "is up to date" in capsys.readouterr().out

    (data / "endpoints.json").write_text(json.dumps({
        **ENDPOINTS,
        "partitions": [dict(p, partitionName="renamed") if p["partition"] == "aws" else p for p in ENDPOINTS["partitions"]],
    }))
    assert r.main(["--check"] + common) == 1
    assert "make aws-data-sync" in capsys.readouterr().out


def test_cli_check_ignores_provenance_differences(tmp_path):
    data, cdk = write_sources(tmp_path)
    output = tmp_path / "partitions.json"
    common = ["--botocore-data", str(data), "--cdk", str(cdk), "--output", str(output)]
    assert r.main(common) == 0
    document = json.loads(output.read_text())
    document["_source"]["botocore"] = "some other label"
    output.write_text(json.dumps(document))
    assert r.main(["--check"] + common) == 0


def test_repo_file_matches_the_local_checkout_when_present():
    if not (r.LOCAL_BOTOCORE_DATA / "partitions.json").exists() or not (r.LOCAL_CDK / r.CDK_ENTITIES).exists():
        pytest.skip("local/aws checkouts not present")
    document = r.generate(r.LOCAL_BOTOCORE_DATA, "test", r.LOCAL_CDK, r.OUTPUT)
    assert r.strip_source(r.render(document)) == r.strip_source(r.OUTPUT.read_text(encoding="utf-8"))