Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
feat(s3files): scope state by region (#1171)
Ian Oberstfbc3909cf46e
diff --git a/ministack/core/persistence.py b/ministack/core/persistence.pyindex 204b4550..226b4fff 100644--- a/ministack/core/persistence.py+++ b/ministack/core/persistence.py@@ -30,6 +30,7 @@ SERVICE_STATE_FORMAT_VERSIONS = {"appsync": 3,"autoscaling": 3,"athena": 3,+ "s3files": 3,"batch": 3,"ecs": 3,"resource_groups": 3,diff --git a/ministack/services/s3files.py b/ministack/services/s3files.pyindex 08185514..0a0cad46 100644--- a/ministack/services/s3files.py+++ b/ministack/services/s3files.py@@ -23,6 +23,7 @@ from urllib.parse import unquotefrom ministack.core.arn import ArnParseError, parse_arnfrom ministack.core.persistence import load_statefrom ministack.core.responses import (+ AccountRegionScopedDict,AccountScopedDict,error_response_json,get_account_id,@@ -43,12 +44,21 @@ def _empty_200():logger = logging.getLogger("s3files")-_file_systems = AccountScopedDict()-_mount_targets = AccountScopedDict()-_access_points = AccountScopedDict()-_policies = AccountScopedDict()-_sync_configs = AccountScopedDict()-_tags = AccountScopedDict()+_file_systems = AccountRegionScopedDict()+_mount_targets = AccountRegionScopedDict()+_access_points = AccountRegionScopedDict()+_policies = AccountRegionScopedDict()+_sync_configs = AccountRegionScopedDict()+_tags = AccountRegionScopedDict()+++def _clear_state():+ _file_systems.clear()+ _mount_targets.clear()+ _access_points.clear()+ _policies.clear()+ _sync_configs.clear()+ _tags.clear()def get_state():@@ -65,12 +75,68 @@ def get_state():def restore_state(data):if not data:return+ _clear_state()_file_systems.update(data.get("file_systems", {}))- _mount_targets.update(data.get("mount_targets", {}))_access_points.update(data.get("access_points", {}))- _policies.update(data.get("policies", {}))- _sync_configs.update(data.get("sync_configs", {}))- _tags.update(data.get("tags", {}))++ resource_regions = {+ (account_id, resource_id): region+ for store in (_file_systems, _access_points)+ for (account_id, region, resource_id), _resource in store.all_items()+ }+ _restore_child_store(+ _mount_targets,+ data.get("mount_targets", {}),+ resource_regions,+ lambda key, value: value.get("fileSystemId", key),+ _mount_target_legacy_region,+ )+ for store, key in (+ (_policies, "policies"),+ (_sync_configs, "sync_configs"),+ (_tags, "tags"),+ ):+ _restore_child_store(+ store,+ data.get(key, {}),+ resource_regions,+ lambda resource_id, _value: resource_id,+ )+++def _mount_target_legacy_region(key, value):+ availability_zone_id = value.get("availabilityZoneId", "")+ region, separator, zone_id = availability_zone_id.rpartition("-az")+ if separator and region and zone_id.isdigit():+ return region+ return _mount_targets._region_for_legacy_value(key, value)+++def _restore_child_store(+ store, restored, resource_regions, parent_id, legacy_region=None+):+ """Adopt legacy child state into its file system or access point region."""+ if isinstance(restored, AccountRegionScopedDict):+ store.update(restored)+ return++ if isinstance(restored, AccountScopedDict):+ items = restored._data.items()+ else:+ account_id = get_account_id()+ items = (((account_id, key), value) for key, value in restored.items())++ for (account_id, key), value in items:+ fallback_region = (+ legacy_region(key, value)+ if legacy_region is not None+ else store._region_for_legacy_value(key, value)+ )+ region = resource_regions.get(+ (account_id, parent_id(key, value)),+ fallback_region,+ )+ store.set_scoped(account_id, region, key, value)try:@@ -82,12 +148,7 @@ except Exception:def reset():- _file_systems.clear()- _mount_targets.clear()- _access_points.clear()- _policies.clear()- _sync_configs.clear()- _tags.clear()+ _clear_state()# ---------------------------------------------------------------------------diff --git a/tests/test_persistence.py b/tests/test_persistence.pyindex 24e8f007..87acb611 100644--- a/tests/test_persistence.py+++ b/tests/test_persistence.py@@ -1868,6 +1868,45 @@ def test_appsync_region_scoped_state_is_rejected_by_v2_reader(assert persistence.load_state("appsync") is None+def test_s3files_region_scoped_state_is_rejected_by_v2_reader(+ monkeypatch, tmp_path+):+ """A rollback binary must reject S3 Files' regional schema instead of+ accepting it as v2 and silently dropping every regional store."""+ import json as _json++ from ministack.core.responses import AccountRegionScopedDict++ monkeypatch.setattr(persistence, "PERSIST_STATE", True)+ monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))++ file_systems = AccountRegionScopedDict()+ file_systems.set_scoped(+ "000000000000",+ "us-west-2",+ "fs-11111111111111111",+ {+ "fileSystemId": "fs-11111111111111111",+ "fileSystemArn": (+ "arn:aws:s3files:us-west-2:000000000000:"+ "file-system/fs-11111111111111111"+ ),+ },+ )+ persistence.save_state("s3files", {"file_systems": file_systems})++ raw = _json.loads((tmp_path / "s3files.json").read_text())+ assert raw["__ministack_format__"] == 3+ loaded_file_systems = persistence.load_state("s3files")["file_systems"]+ assert loaded_file_systems.get_scoped(+ "000000000000", "us-west-2", "fs-11111111111111111"+ )["fileSystemId"] == "fs-11111111111111111"++ # Simulate the previous binary, whose highest understood format is v2.+ monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})+ assert persistence.load_state("s3files") is None++def test_resource_groups_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):diff --git a/tests/test_s3files.py b/tests/test_s3files.pyindex 91433cb8..0fe5d76c 100644--- a/tests/test_s3files.py+++ b/tests/test_s3files.py@@ -17,23 +17,20 @@ BUCKET_ARN = "arn:aws:s3:::test-bucket"ROLE_ARN = "arn:aws:iam::000000000000:role/s3files-role"-def _auth(account="000000000000"):+def _auth(account="000000000000", region="us-east-1"):return (f"AWS4-HMAC-SHA256 "- f"Credential={account}/20260504/us-east-1/s3files/aws4_request, "+ f"Credential={account}/20260504/{region}/s3files/aws4_request, "f"SignedHeaders=host, Signature=00")-_AUTH = _auth()---def _req(method, path, body=None, query=None, account=None):+def _req(method, path, body=None, query=None, account=None, region="us-east-1"):url = ENDPOINT + pathif query:url += "?" + urllib.parse.urlencode(query, doseq=True)data = None- headers = {"Authorization": _auth(account) if account else _AUTH}+ headers = {"Authorization": _auth(account or "000000000000", region)}if body is not None:data = json.dumps(body).encode()headers["Content-Type"] = "application/json"@@ -86,6 +83,191 @@ def test_create_file_system_uses_put_and_camel_case():_req("DELETE", f"/file-systems/{fs_id}")+def test_file_systems_are_region_scoped():+ _, east_fs = _req(+ "PUT",+ "/file-systems",+ {"bucket": BUCKET_ARN, "roleArn": ROLE_ARN},+ region="us-east-1",+ )+ _, west_fs = _req(+ "PUT",+ "/file-systems",+ {"bucket": BUCKET_ARN, "roleArn": ROLE_ARN},+ region="us-west-2",+ )+ try:+ _, east_list = _req("GET", "/file-systems", region="us-east-1")+ _, west_list = _req("GET", "/file-systems", region="us-west-2")+ east_ids = {fs["fileSystemId"] for fs in east_list["fileSystems"]}+ west_ids = {fs["fileSystemId"] for fs in west_list["fileSystems"]}++ assert east_fs["fileSystemId"] in east_ids+ assert east_fs["fileSystemId"] not in west_ids+ assert west_fs["fileSystemId"] in west_ids+ assert west_fs["fileSystemId"] not in east_ids+ assert ":us-east-1:" in east_fs["fileSystemArn"]+ assert ":us-west-2:" in west_fs["fileSystemArn"]+ finally:+ _req(+ "DELETE",+ f"/file-systems/{east_fs['fileSystemId']}",+ region="us-east-1",+ )+ _req(+ "DELETE",+ f"/file-systems/{west_fs['fileSystemId']}",+ region="us-west-2",+ )+++def test_legacy_child_state_restores_beside_parent_resource():+ from ministack.core.responses import (+ AccountScopedDict,+ get_account_id,+ get_region,+ set_request_account_id,+ set_request_region,+ )+ from ministack.services import s3files as service++ original_account = get_account_id()+ original_region = get_region()+ account_id = "111111111111"+ boot_region = "us-east-1"+ resource_region = "us-west-2"+ fs_id = "fs-11111111111111111"+ mt_id = "fsmt-11111111111111111"+ ap_id = "fsap-11111111111111111"+ file_systems = AccountScopedDict()+ mount_targets = AccountScopedDict()+ access_points = AccountScopedDict()++ set_request_account_id(account_id)+ set_request_region(boot_region)+ file_systems[fs_id] = {+ "fileSystemId": fs_id,+ "fileSystemArn": (+ f"arn:aws:s3files:{resource_region}:{account_id}:file-system/{fs_id}"+ ),+ }+ mount_targets[mt_id] = {"mountTargetId": mt_id, "fileSystemId": fs_id}+ access_points[ap_id] = {+ "accessPointId": ap_id,+ "fileSystemId": fs_id,+ "accessPointArn": (+ f"arn:aws:s3files:{resource_region}:{account_id}:"+ f"file-system/{fs_id}/access-point/{ap_id}"+ ),+ }+ payload = {+ "file_systems": file_systems,+ "mount_targets": mount_targets,+ "access_points": access_points,+ }+ for state_key, resource_id, value in (+ ("policies", fs_id, "legacy-policy"),+ ("sync_configs", fs_id, {"latestVersionNumber": 1}),+ ("tags", ap_id, [{"key": "legacy", "value": "true"}]),+ ):+ store = AccountScopedDict()+ store[resource_id] = value+ payload[state_key] = store++ service.reset()+ try:+ service.restore_state(payload)+ assert service._file_systems.get_scoped(+ account_id, resource_region, fs_id+ )["fileSystemId"] == fs_id+ assert service._mount_targets.get_scoped(+ account_id, resource_region, mt_id+ )["fileSystemId"] == fs_id+ assert service._access_points.get_scoped(+ account_id, resource_region, ap_id+ )["fileSystemId"] == fs_id+ assert service._policies.get_scoped(+ account_id, resource_region, fs_id+ ) == "legacy-policy"+ assert service._sync_configs.get_scoped(+ account_id, resource_region, fs_id+ ) == {"latestVersionNumber": 1}+ assert service._tags.get_scoped(account_id, resource_region, ap_id) == [+ {"key": "legacy", "value": "true"}+ ]+ finally:+ service.reset()+ set_request_account_id(original_account)+ set_request_region(original_region)+++def test_legacy_orphaned_mount_target_restores_to_availability_zone_region():+ from ministack.core.responses import (+ AccountScopedDict,+ get_account_id,+ get_region,+ set_request_account_id,+ set_request_region,+ )+ from ministack.services import s3files as service++ original_account = get_account_id()+ original_region = get_region()+ account_id = "111111111111"+ boot_region = "us-east-1"+ resource_region = "us-west-2"+ mt_id = "fsmt-11111111111111111"+ mount_targets = AccountScopedDict()++ set_request_account_id(account_id)+ set_request_region(boot_region)+ mount_targets[mt_id] = {+ "mountTargetId": mt_id,+ "fileSystemId": "fs-11111111111111111",+ "availabilityZoneId": f"{resource_region}-az1",+ }++ service.reset()+ try:+ service.restore_state({"mount_targets": mount_targets})+ assert service._mount_targets.get_scoped(+ account_id, resource_region, mt_id+ )["mountTargetId"] == mt_id+ assert (+ service._mount_targets.get_scoped(account_id, boot_region, mt_id) is None+ )+ finally:+ service.reset()+ set_request_account_id(original_account)+ set_request_region(original_region)+++def test_reset_clears_s3files_state_across_regions():+ from ministack.core.responses import get_region, set_request_region+ from ministack.services import s3files as service++ original_region = get_region()+ stores = (+ service._file_systems,+ service._mount_targets,+ service._access_points,+ service._policies,+ service._sync_configs,+ service._tags,+ )+ service.reset()+ try:+ for region in ("us-east-1", "us-west-2"):+ set_request_region(region)+ for store in stores:+ store[f"resource-{region}"] = {"region": region}+ service.reset()+ assert all(not store.has_any() for store in stores)+ finally:+ service.reset()+ set_request_region(original_region)++def test_create_file_system_post_is_not_supported():status, body = _req("POST", "/file-systems", {"bucket": BUCKET_ARN,