Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
feat(emr): scope state by region (#1178)
Ian Oberstef6f1939ead9
diff --git a/ministack/core/persistence.py b/ministack/core/persistence.pyindex 2576d7d2..831c82fd 100644--- a/ministack/core/persistence.py+++ b/ministack/core/persistence.py@@ -30,6 +30,7 @@ SERVICE_STATE_FORMAT_VERSIONS = {"appsync": 3,"autoscaling": 3,"athena": 3,+ "emr": 3,"inspector2": 3,"efs": 3,"s3files": 3,diff --git a/ministack/services/emr.py b/ministack/services/emr.pyindex e4264747..fe26bc25 100644--- a/ministack/services/emr.py+++ b/ministack/services/emr.py@@ -24,6 +24,7 @@ import timefrom ministack.core.persistence import PERSIST_STATE, load_statefrom ministack.core.responses import (+ AccountRegionScopedDict,AccountScopedDict,error_response_json,get_account_id,@@ -40,12 +41,29 @@ REGION = os.environ.get("MINISTACK_REGION", "us-east-1")# State# ----------------------------------------------------------------------------_clusters = AccountScopedDict() # cluster_id -> cluster record-_steps = AccountScopedDict() # cluster_id -> [step records]-_block_public_access: dict = {- "BlockPublicSecurityGroupRules": False,- "PermittedPublicSecurityGroupRuleRanges": [],-}+_clusters = AccountRegionScopedDict() # cluster_id -> cluster record+_steps = AccountRegionScopedDict() # cluster_id -> [step records]+_block_public_access = AccountRegionScopedDict()+_BLOCK_PUBLIC_ACCESS_KEY = "configuration"+++def _default_block_public_access():+ return {+ "BlockPublicSecurityGroupRules": False,+ "PermittedPublicSecurityGroupRuleRanges": [],+ }+++def _get_block_public_access():+ if _BLOCK_PUBLIC_ACCESS_KEY not in _block_public_access:+ _block_public_access[_BLOCK_PUBLIC_ACCESS_KEY] = _default_block_public_access()+ return _block_public_access[_BLOCK_PUBLIC_ACCESS_KEY]+++def _clear_state():+ _clusters.clear()+ _steps.clear()+ _block_public_access.clear()def get_state():@@ -57,9 +75,59 @@ def get_state():def restore_state(data):+ if not data:+ return+ _clear_state()_clusters.update(data.get("_clusters", {}))- _steps.update(data.get("_steps", {}))- _block_public_access.update(data.get("_block_public_access", {}))+ cluster_regions = {+ (account_id, cluster_id): region+ for (account_id, region, cluster_id), _cluster in _clusters.all_items()+ }+ _restore_steps(data.get("_steps", {}), cluster_regions)+ _restore_block_public_access(data.get("_block_public_access", {}))+++def _restore_steps(restored, cluster_regions):+ """Adopt legacy step lists into their parent cluster's region."""+ if isinstance(restored, AccountRegionScopedDict):+ _steps.update(restored)+ return++ if isinstance(restored, AccountScopedDict):+ items = restored._data.items()+ else:+ account_id = get_account_id()+ items = (((account_id, key), value) for key, value in restored.items())++ for (account_id, cluster_id), value in items:+ # Step arguments can contain unrelated regional ARNs, so an orphaned+ # step list stays in the boot region instead of inferring from content.+ region = cluster_regions.get(+ (account_id, cluster_id),+ get_region(),+ )+ _steps.set_scoped(account_id, region, cluster_id, value)+++def _restore_block_public_access(restored):+ if isinstance(restored, AccountRegionScopedDict):+ _block_public_access.update(restored)+ return++ if isinstance(restored, AccountScopedDict):+ for (account_id, key), value in restored._data.items():+ _block_public_access.set_scoped(account_id, get_region(), key, value)+ return++ if "BlockPublicSecurityGroupRules" in restored:+ _block_public_access[_BLOCK_PUBLIC_ACCESS_KEY] = {+ "BlockPublicSecurityGroupRules": restored.get(+ "BlockPublicSecurityGroupRules", False+ ),+ "PermittedPublicSecurityGroupRuleRanges": restored.get(+ "PermittedPublicSecurityGroupRuleRanges", []+ ),+ }try:@@ -537,7 +605,7 @@ def _remove_tags(data):def _get_block_public_access_configuration(data):return json_response({- "BlockPublicAccessConfiguration": _block_public_access,+ "BlockPublicAccessConfiguration": _get_block_public_access(),"BlockPublicAccessConfigurationMetadata": {"CreationDateTime": _now_iso(),"CreatedByArn": f"arn:aws:iam::{get_account_id()}:root",@@ -547,10 +615,11 @@ def _get_block_public_access_configuration(data):def _put_block_public_access_configuration(data):config = data.get("BlockPublicAccessConfiguration", {})- _block_public_access["BlockPublicSecurityGroupRules"] = config.get(+ scoped_config = _get_block_public_access()+ scoped_config["BlockPublicSecurityGroupRules"] = config.get("BlockPublicSecurityGroupRules", False)- _block_public_access["PermittedPublicSecurityGroupRuleRanges"] = config.get(+ scoped_config["PermittedPublicSecurityGroupRuleRanges"] = config.get("PermittedPublicSecurityGroupRuleRanges", [])return json_response({})@@ -606,7 +675,4 @@ async def handle_request(method, path, headers, body, query_params):# ---------------------------------------------------------------------------def reset():- _clusters.clear()- _steps.clear()- _block_public_access["BlockPublicSecurityGroupRules"] = False- _block_public_access["PermittedPublicSecurityGroupRuleRanges"] = []+ _clear_state()diff --git a/tests/test_emr.py b/tests/test_emr.pyindex 159b7794..e99adcb3 100644--- a/tests/test_emr.py+++ b/tests/test_emr.py@@ -6,10 +6,180 @@ import uuid as _uuid_modimport zipfilefrom urllib.parse import urlparse+import boto3import pytest+from botocore.config import Configfrom botocore.exceptions import ClientError+def _emr_client(region):+ return boto3.client(+ "emr",+ endpoint_url=os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566"),+ aws_access_key_id="test",+ aws_secret_access_key="test",+ region_name=region,+ config=Config(region_name=region, retries={"mode": "standard"}),+ )+++def _run_regional_cluster(client, name):+ return client.run_job_flow(+ Name=name,+ ReleaseLabel="emr-6.10.0",+ Instances={+ "MasterInstanceType": "m5.xlarge",+ "InstanceCount": 1,+ "KeepJobFlowAliveWhenNoSteps": True,+ },+ JobFlowRole="EMR_EC2_DefaultRole",+ ServiceRole="EMR_DefaultRole",+ )+++def test_emr_state_and_block_public_access_are_region_scoped():+ east = _emr_client("us-east-1")+ west = _emr_client("us-west-2")+ east_cluster = _run_regional_cluster(east, "same-name-regional-cluster")+ west_cluster = _run_regional_cluster(west, "same-name-regional-cluster")+ east_step = east.add_job_flow_steps(+ JobFlowId=east_cluster["JobFlowId"],+ Steps=[{"Name": "east-step", "HadoopJarStep": {"Jar": "command-runner.jar"}}],+ )["StepIds"][0]+ west_step = west.add_job_flow_steps(+ JobFlowId=west_cluster["JobFlowId"],+ Steps=[{"Name": "west-step", "HadoopJarStep": {"Jar": "command-runner.jar"}}],+ )["StepIds"][0]++ try:+ east_ids = {cluster["Id"] for cluster in east.list_clusters()["Clusters"]}+ west_ids = {cluster["Id"] for cluster in west.list_clusters()["Clusters"]}+ assert east_cluster["JobFlowId"] in east_ids+ assert east_cluster["JobFlowId"] not in west_ids+ assert west_cluster["JobFlowId"] in west_ids+ assert west_cluster["JobFlowId"] not in east_ids+ assert ":us-east-1:" in east_cluster["ClusterArn"]+ assert ":us-west-2:" in west_cluster["ClusterArn"]+ assert east.list_steps(ClusterId=east_cluster["JobFlowId"])["Steps"][0]["Id"] == east_step+ assert west.list_steps(ClusterId=west_cluster["JobFlowId"])["Steps"][0]["Id"] == west_step++ assert east.get_block_public_access_configuration()["BlockPublicAccessConfiguration"] == {+ "BlockPublicSecurityGroupRules": False,+ "PermittedPublicSecurityGroupRuleRanges": [],+ }+ assert west.get_block_public_access_configuration()["BlockPublicAccessConfiguration"] == {+ "BlockPublicSecurityGroupRules": False,+ "PermittedPublicSecurityGroupRuleRanges": [],+ }+ east.put_block_public_access_configuration(+ BlockPublicAccessConfiguration={+ "BlockPublicSecurityGroupRules": True,+ "PermittedPublicSecurityGroupRuleRanges": [{"MinRange": 22, "MaxRange": 22}],+ }+ )+ assert east.get_block_public_access_configuration()["BlockPublicAccessConfiguration"][+ "BlockPublicSecurityGroupRules"+ ] is True+ assert west.get_block_public_access_configuration()["BlockPublicAccessConfiguration"] == {+ "BlockPublicSecurityGroupRules": False,+ "PermittedPublicSecurityGroupRuleRanges": [],+ }+ finally:+ east.put_block_public_access_configuration(+ BlockPublicAccessConfiguration={"BlockPublicSecurityGroupRules": False}+ )+ west.put_block_public_access_configuration(+ BlockPublicAccessConfiguration={"BlockPublicSecurityGroupRules": False}+ )+ east.terminate_job_flows(JobFlowIds=[east_cluster["JobFlowId"]])+ west.terminate_job_flows(JobFlowIds=[west_cluster["JobFlowId"]])+++def test_emr_legacy_state_restores_children_beside_parent():+ from ministack.core.responses import (+ AccountScopedDict,+ get_account_id,+ get_region,+ set_request_account_id,+ set_request_region,+ )+ from ministack.services import emr as service++ original_account = get_account_id()+ original_region = get_region()+ account_id = "111111111111"+ boot_region = "us-east-1"+ cluster_region = "us-west-2"+ cluster_id = "j-LEGACYCLUSTER"+ orphan_id = "j-ORPHANCLUSTER"+ clusters = AccountScopedDict()+ steps = AccountScopedDict()++ set_request_account_id(account_id)+ set_request_region(boot_region)+ clusters[cluster_id] = {+ "Id": cluster_id,+ "ClusterArn": f"arn:aws:elasticmapreduce:{cluster_region}:{account_id}:cluster/{cluster_id}",+ }+ steps[cluster_id] = [{"Id": "s-LEGACY", "Name": "legacy-step"}]+ steps[orphan_id] = [+ {+ "Id": "s-ORPHAN",+ "Name": "orphan-step",+ "Config": {+ "Args": [+ f"arn:aws:lambda:{cluster_region}:{account_id}:function:incidental"+ ]+ },+ }+ ]++ service.reset()+ try:+ service.restore_state(+ {+ "_clusters": clusters,+ "_steps": steps,+ "_block_public_access": {+ "BlockPublicSecurityGroupRules": True,+ "PermittedPublicSecurityGroupRuleRanges": [{"MinRange": 22, "MaxRange": 22}],+ },+ }+ )+ assert service._clusters.get_scoped(account_id, cluster_region, cluster_id)["Id"] == cluster_id+ assert service._steps.get_scoped(account_id, cluster_region, cluster_id)[0]["Id"] == "s-LEGACY"+ assert service._steps.get_scoped(account_id, boot_region, orphan_id)[0]["Id"] == "s-ORPHAN"+ assert service._block_public_access.get_scoped(+ account_id, boot_region, service._BLOCK_PUBLIC_ACCESS_KEY+ )["BlockPublicSecurityGroupRules"] is True+ assert service._block_public_access.get_scoped(+ account_id, cluster_region, service._BLOCK_PUBLIC_ACCESS_KEY+ ) is None+ finally:+ service.reset()+ set_request_account_id(original_account)+ set_request_region(original_region)+++def test_emr_reset_clears_state_across_regions():+ from ministack.core.responses import get_region, set_request_region+ from ministack.services import emr as service++ original_region = get_region()+ stores = (service._clusters, service._steps, service._block_public_access)+ service.reset()+ try:+ for region in ("us-east-1", "us-west-2"):+ set_request_region(region)+ for store in stores:+ store[f"resource-{region}"] = {"region": region}+ service.reset()+ assert all(not store.has_any() for store in stores)+ finally:+ service.reset()+ set_request_region(original_region)++def test_emr_run_job_flow_simple(emr):resp = emr.run_job_flow(Name="test-cluster",diff --git a/tests/test_persistence.py b/tests/test_persistence.pyindex 3c2ee540..70152aeb 100644--- a/tests/test_persistence.py+++ b/tests/test_persistence.py@@ -1868,10 +1868,8 @@ def test_appsync_region_scoped_state_is_rejected_by_v2_reader(assert persistence.load_state("appsync") is None-def test_inspector2_region_scoped_state_is_rejected_by_v2_reader(- monkeypatch, tmp_path-):- """A rollback binary must reject Inspector2 regional account buckets."""+def test_emr_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):+ """A rollback binary must reject EMR regional state instead of dropping it."""import json as _jsonfrom ministack.core.responses import AccountRegionScopedDict@@ -1879,24 +1877,30 @@ def test_inspector2_region_scoped_state_is_rejected_by_v2_reader(monkeypatch.setattr(persistence, "PERSIST_STATE", True)monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))- configs = AccountRegionScopedDict()- configs.set_scoped(+ clusters = AccountRegionScopedDict()+ clusters.set_scoped("000000000000","us-west-2",- "000000000000",- {"ecr": {"status": "ENABLED"}},+ "j-LEGACYCLUSTER",+ {+ "Id": "j-LEGACYCLUSTER",+ "ClusterArn": (+ "arn:aws:elasticmapreduce:us-west-2:000000000000:"+ "cluster/j-LEGACYCLUSTER"+ ),+ },)- persistence.save_state("inspector2", {"account_config": configs})+ persistence.save_state("emr", {"_clusters": clusters})- raw = _json.loads((tmp_path / "inspector2.json").read_text())+ raw = _json.loads((tmp_path / "emr.json").read_text())assert raw["__ministack_format__"] == 3- loaded = persistence.load_state("inspector2")["account_config"]+ loaded = persistence.load_state("emr")["_clusters"]assert loaded.get_scoped(- "000000000000", "us-west-2", "000000000000"- ) == {"ecr": {"status": "ENABLED"}}+ "000000000000", "us-west-2", "j-LEGACYCLUSTER"+ )["Id"] == "j-LEGACYCLUSTER"monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})- assert persistence.load_state("inspector2") is None+ assert persistence.load_state("emr") is Nonedef test_resource_groups_region_scoped_state_is_rejected_by_v2_reader(