Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

feat(emr): scope state by region (#1178)

Ian Oberstef6f1939ead9
diff --git a/ministack/core/persistence.py b/ministack/core/persistence.py
index 2576d7d2..831c82fd 100644
--- a/ministack/core/persistence.py
+++ b/ministack/core/persistence.py
@@ -30,6 +30,7 @@ SERVICE_STATE_FORMAT_VERSIONS = {
"appsync": 3,
"autoscaling": 3,
"athena": 3,
+ "emr": 3,
"inspector2": 3,
"efs": 3,
"s3files": 3,
diff --git a/ministack/services/emr.py b/ministack/services/emr.py
index e4264747..fe26bc25 100644
--- a/ministack/services/emr.py
+++ b/ministack/services/emr.py
@@ -24,6 +24,7 @@ import time
from ministack.core.persistence import PERSIST_STATE, load_state
from ministack.core.responses import (
+ AccountRegionScopedDict,
AccountScopedDict,
error_response_json,
get_account_id,
@@ -40,12 +41,29 @@ REGION = os.environ.get("MINISTACK_REGION", "us-east-1")
# State
# ---------------------------------------------------------------------------
-_clusters = AccountScopedDict() # cluster_id -> cluster record
-_steps = AccountScopedDict() # cluster_id -> [step records]
-_block_public_access: dict = {
- "BlockPublicSecurityGroupRules": False,
- "PermittedPublicSecurityGroupRuleRanges": [],
-}
+_clusters = AccountRegionScopedDict() # cluster_id -> cluster record
+_steps = AccountRegionScopedDict() # cluster_id -> [step records]
+_block_public_access = AccountRegionScopedDict()
+_BLOCK_PUBLIC_ACCESS_KEY = "configuration"
+
+
+def _default_block_public_access():
+ return {
+ "BlockPublicSecurityGroupRules": False,
+ "PermittedPublicSecurityGroupRuleRanges": [],
+ }
+
+
+def _get_block_public_access():
+ if _BLOCK_PUBLIC_ACCESS_KEY not in _block_public_access:
+ _block_public_access[_BLOCK_PUBLIC_ACCESS_KEY] = _default_block_public_access()
+ return _block_public_access[_BLOCK_PUBLIC_ACCESS_KEY]
+
+
+def _clear_state():
+ _clusters.clear()
+ _steps.clear()
+ _block_public_access.clear()
def get_state():
@@ -57,9 +75,59 @@ def get_state():
def restore_state(data):
+ if not data:
+ return
+ _clear_state()
_clusters.update(data.get("_clusters", {}))
- _steps.update(data.get("_steps", {}))
- _block_public_access.update(data.get("_block_public_access", {}))
+ cluster_regions = {
+ (account_id, cluster_id): region
+ for (account_id, region, cluster_id), _cluster in _clusters.all_items()
+ }
+ _restore_steps(data.get("_steps", {}), cluster_regions)
+ _restore_block_public_access(data.get("_block_public_access", {}))
+
+
+def _restore_steps(restored, cluster_regions):
+ """Adopt legacy step lists into their parent cluster's region."""
+ if isinstance(restored, AccountRegionScopedDict):
+ _steps.update(restored)
+ return
+
+ if isinstance(restored, AccountScopedDict):
+ items = restored._data.items()
+ else:
+ account_id = get_account_id()
+ items = (((account_id, key), value) for key, value in restored.items())
+
+ for (account_id, cluster_id), value in items:
+ # Step arguments can contain unrelated regional ARNs, so an orphaned
+ # step list stays in the boot region instead of inferring from content.
+ region = cluster_regions.get(
+ (account_id, cluster_id),
+ get_region(),
+ )
+ _steps.set_scoped(account_id, region, cluster_id, value)
+
+
+def _restore_block_public_access(restored):
+ if isinstance(restored, AccountRegionScopedDict):
+ _block_public_access.update(restored)
+ return
+
+ if isinstance(restored, AccountScopedDict):
+ for (account_id, key), value in restored._data.items():
+ _block_public_access.set_scoped(account_id, get_region(), key, value)
+ return
+
+ if "BlockPublicSecurityGroupRules" in restored:
+ _block_public_access[_BLOCK_PUBLIC_ACCESS_KEY] = {
+ "BlockPublicSecurityGroupRules": restored.get(
+ "BlockPublicSecurityGroupRules", False
+ ),
+ "PermittedPublicSecurityGroupRuleRanges": restored.get(
+ "PermittedPublicSecurityGroupRuleRanges", []
+ ),
+ }
try:
@@ -537,7 +605,7 @@ def _remove_tags(data):
def _get_block_public_access_configuration(data):
return json_response({
- "BlockPublicAccessConfiguration": _block_public_access,
+ "BlockPublicAccessConfiguration": _get_block_public_access(),
"BlockPublicAccessConfigurationMetadata": {
"CreationDateTime": _now_iso(),
"CreatedByArn": f"arn:aws:iam::{get_account_id()}:root",
@@ -547,10 +615,11 @@ def _get_block_public_access_configuration(data):
def _put_block_public_access_configuration(data):
config = data.get("BlockPublicAccessConfiguration", {})
- _block_public_access["BlockPublicSecurityGroupRules"] = config.get(
+ scoped_config = _get_block_public_access()
+ scoped_config["BlockPublicSecurityGroupRules"] = config.get(
"BlockPublicSecurityGroupRules", False
)
- _block_public_access["PermittedPublicSecurityGroupRuleRanges"] = config.get(
+ scoped_config["PermittedPublicSecurityGroupRuleRanges"] = config.get(
"PermittedPublicSecurityGroupRuleRanges", []
)
return json_response({})
@@ -606,7 +675,4 @@ async def handle_request(method, path, headers, body, query_params):
# ---------------------------------------------------------------------------
def reset():
- _clusters.clear()
- _steps.clear()
- _block_public_access["BlockPublicSecurityGroupRules"] = False
- _block_public_access["PermittedPublicSecurityGroupRuleRanges"] = []
+ _clear_state()
diff --git a/tests/test_emr.py b/tests/test_emr.py
index 159b7794..e99adcb3 100644
--- a/tests/test_emr.py
+++ b/tests/test_emr.py
@@ -6,10 +6,180 @@ import uuid as _uuid_mod
import zipfile
from urllib.parse import urlparse
+import boto3
import pytest
+from botocore.config import Config
from botocore.exceptions import ClientError
+def _emr_client(region):
+ return boto3.client(
+ "emr",
+ endpoint_url=os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566"),
+ aws_access_key_id="test",
+ aws_secret_access_key="test",
+ region_name=region,
+ config=Config(region_name=region, retries={"mode": "standard"}),
+ )
+
+
+def _run_regional_cluster(client, name):
+ return client.run_job_flow(
+ Name=name,
+ ReleaseLabel="emr-6.10.0",
+ Instances={
+ "MasterInstanceType": "m5.xlarge",
+ "InstanceCount": 1,
+ "KeepJobFlowAliveWhenNoSteps": True,
+ },
+ JobFlowRole="EMR_EC2_DefaultRole",
+ ServiceRole="EMR_DefaultRole",
+ )
+
+
+def test_emr_state_and_block_public_access_are_region_scoped():
+ east = _emr_client("us-east-1")
+ west = _emr_client("us-west-2")
+ east_cluster = _run_regional_cluster(east, "same-name-regional-cluster")
+ west_cluster = _run_regional_cluster(west, "same-name-regional-cluster")
+ east_step = east.add_job_flow_steps(
+ JobFlowId=east_cluster["JobFlowId"],
+ Steps=[{"Name": "east-step", "HadoopJarStep": {"Jar": "command-runner.jar"}}],
+ )["StepIds"][0]
+ west_step = west.add_job_flow_steps(
+ JobFlowId=west_cluster["JobFlowId"],
+ Steps=[{"Name": "west-step", "HadoopJarStep": {"Jar": "command-runner.jar"}}],
+ )["StepIds"][0]
+
+ try:
+ east_ids = {cluster["Id"] for cluster in east.list_clusters()["Clusters"]}
+ west_ids = {cluster["Id"] for cluster in west.list_clusters()["Clusters"]}
+ assert east_cluster["JobFlowId"] in east_ids
+ assert east_cluster["JobFlowId"] not in west_ids
+ assert west_cluster["JobFlowId"] in west_ids
+ assert west_cluster["JobFlowId"] not in east_ids
+ assert ":us-east-1:" in east_cluster["ClusterArn"]
+ assert ":us-west-2:" in west_cluster["ClusterArn"]
+ assert east.list_steps(ClusterId=east_cluster["JobFlowId"])["Steps"][0]["Id"] == east_step
+ assert west.list_steps(ClusterId=west_cluster["JobFlowId"])["Steps"][0]["Id"] == west_step
+
+ assert east.get_block_public_access_configuration()["BlockPublicAccessConfiguration"] == {
+ "BlockPublicSecurityGroupRules": False,
+ "PermittedPublicSecurityGroupRuleRanges": [],
+ }
+ assert west.get_block_public_access_configuration()["BlockPublicAccessConfiguration"] == {
+ "BlockPublicSecurityGroupRules": False,
+ "PermittedPublicSecurityGroupRuleRanges": [],
+ }
+ east.put_block_public_access_configuration(
+ BlockPublicAccessConfiguration={
+ "BlockPublicSecurityGroupRules": True,
+ "PermittedPublicSecurityGroupRuleRanges": [{"MinRange": 22, "MaxRange": 22}],
+ }
+ )
+ assert east.get_block_public_access_configuration()["BlockPublicAccessConfiguration"][
+ "BlockPublicSecurityGroupRules"
+ ] is True
+ assert west.get_block_public_access_configuration()["BlockPublicAccessConfiguration"] == {
+ "BlockPublicSecurityGroupRules": False,
+ "PermittedPublicSecurityGroupRuleRanges": [],
+ }
+ finally:
+ east.put_block_public_access_configuration(
+ BlockPublicAccessConfiguration={"BlockPublicSecurityGroupRules": False}
+ )
+ west.put_block_public_access_configuration(
+ BlockPublicAccessConfiguration={"BlockPublicSecurityGroupRules": False}
+ )
+ east.terminate_job_flows(JobFlowIds=[east_cluster["JobFlowId"]])
+ west.terminate_job_flows(JobFlowIds=[west_cluster["JobFlowId"]])
+
+
+def test_emr_legacy_state_restores_children_beside_parent():
+ from ministack.core.responses import (
+ AccountScopedDict,
+ get_account_id,
+ get_region,
+ set_request_account_id,
+ set_request_region,
+ )
+ from ministack.services import emr as service
+
+ original_account = get_account_id()
+ original_region = get_region()
+ account_id = "111111111111"
+ boot_region = "us-east-1"
+ cluster_region = "us-west-2"
+ cluster_id = "j-LEGACYCLUSTER"
+ orphan_id = "j-ORPHANCLUSTER"
+ clusters = AccountScopedDict()
+ steps = AccountScopedDict()
+
+ set_request_account_id(account_id)
+ set_request_region(boot_region)
+ clusters[cluster_id] = {
+ "Id": cluster_id,
+ "ClusterArn": f"arn:aws:elasticmapreduce:{cluster_region}:{account_id}:cluster/{cluster_id}",
+ }
+ steps[cluster_id] = [{"Id": "s-LEGACY", "Name": "legacy-step"}]
+ steps[orphan_id] = [
+ {
+ "Id": "s-ORPHAN",
+ "Name": "orphan-step",
+ "Config": {
+ "Args": [
+ f"arn:aws:lambda:{cluster_region}:{account_id}:function:incidental"
+ ]
+ },
+ }
+ ]
+
+ service.reset()
+ try:
+ service.restore_state(
+ {
+ "_clusters": clusters,
+ "_steps": steps,
+ "_block_public_access": {
+ "BlockPublicSecurityGroupRules": True,
+ "PermittedPublicSecurityGroupRuleRanges": [{"MinRange": 22, "MaxRange": 22}],
+ },
+ }
+ )
+ assert service._clusters.get_scoped(account_id, cluster_region, cluster_id)["Id"] == cluster_id
+ assert service._steps.get_scoped(account_id, cluster_region, cluster_id)[0]["Id"] == "s-LEGACY"
+ assert service._steps.get_scoped(account_id, boot_region, orphan_id)[0]["Id"] == "s-ORPHAN"
+ assert service._block_public_access.get_scoped(
+ account_id, boot_region, service._BLOCK_PUBLIC_ACCESS_KEY
+ )["BlockPublicSecurityGroupRules"] is True
+ assert service._block_public_access.get_scoped(
+ account_id, cluster_region, service._BLOCK_PUBLIC_ACCESS_KEY
+ ) is None
+ finally:
+ service.reset()
+ set_request_account_id(original_account)
+ set_request_region(original_region)
+
+
+def test_emr_reset_clears_state_across_regions():
+ from ministack.core.responses import get_region, set_request_region
+ from ministack.services import emr as service
+
+ original_region = get_region()
+ stores = (service._clusters, service._steps, service._block_public_access)
+ service.reset()
+ try:
+ for region in ("us-east-1", "us-west-2"):
+ set_request_region(region)
+ for store in stores:
+ store[f"resource-{region}"] = {"region": region}
+ service.reset()
+ assert all(not store.has_any() for store in stores)
+ finally:
+ service.reset()
+ set_request_region(original_region)
+
+
def test_emr_run_job_flow_simple(emr):
resp = emr.run_job_flow(
Name="test-cluster",
diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index 3c2ee540..70152aeb 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -1868,10 +1868,8 @@ def test_appsync_region_scoped_state_is_rejected_by_v2_reader(
assert persistence.load_state("appsync") is None
-def test_inspector2_region_scoped_state_is_rejected_by_v2_reader(
- monkeypatch, tmp_path
-):
- """A rollback binary must reject Inspector2 regional account buckets."""
+def test_emr_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
+ """A rollback binary must reject EMR regional state instead of dropping it."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
@@ -1879,24 +1877,30 @@ def test_inspector2_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
- configs = AccountRegionScopedDict()
- configs.set_scoped(
+ clusters = AccountRegionScopedDict()
+ clusters.set_scoped(
"000000000000",
"us-west-2",
- "000000000000",
- {"ecr": {"status": "ENABLED"}},
+ "j-LEGACYCLUSTER",
+ {
+ "Id": "j-LEGACYCLUSTER",
+ "ClusterArn": (
+ "arn:aws:elasticmapreduce:us-west-2:000000000000:"
+ "cluster/j-LEGACYCLUSTER"
+ ),
+ },
)
- persistence.save_state("inspector2", {"account_config": configs})
+ persistence.save_state("emr", {"_clusters": clusters})
- raw = _json.loads((tmp_path / "inspector2.json").read_text())
+ raw = _json.loads((tmp_path / "emr.json").read_text())
assert raw["__ministack_format__"] == 3
- loaded = persistence.load_state("inspector2")["account_config"]
+ loaded = persistence.load_state("emr")["_clusters"]
assert loaded.get_scoped(
- "000000000000", "us-west-2", "000000000000"
- ) == {"ecr": {"status": "ENABLED"}}
+ "000000000000", "us-west-2", "j-LEGACYCLUSTER"
+ )["Id"] == "j-LEGACYCLUSTER"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
- assert persistence.load_state("inspector2") is None
+ assert persistence.load_state("emr") is None
def test_resource_groups_region_scoped_state_is_rejected_by_v2_reader(