Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

feat(s3files): scope state by region (#1171)

Ian Oberstfbc3909cf46e
diff --git a/ministack/core/persistence.py b/ministack/core/persistence.py
index 204b4550..226b4fff 100644
--- a/ministack/core/persistence.py
+++ b/ministack/core/persistence.py
@@ -30,6 +30,7 @@ SERVICE_STATE_FORMAT_VERSIONS = {
"appsync": 3,
"autoscaling": 3,
"athena": 3,
+ "s3files": 3,
"batch": 3,
"ecs": 3,
"resource_groups": 3,
diff --git a/ministack/services/s3files.py b/ministack/services/s3files.py
index 08185514..0a0cad46 100644
--- a/ministack/services/s3files.py
+++ b/ministack/services/s3files.py
@@ -23,6 +23,7 @@ from urllib.parse import unquote
from ministack.core.arn import ArnParseError, parse_arn
from ministack.core.persistence import load_state
from ministack.core.responses import (
+ AccountRegionScopedDict,
AccountScopedDict,
error_response_json,
get_account_id,
@@ -43,12 +44,21 @@ def _empty_200():
logger = logging.getLogger("s3files")
-_file_systems = AccountScopedDict()
-_mount_targets = AccountScopedDict()
-_access_points = AccountScopedDict()
-_policies = AccountScopedDict()
-_sync_configs = AccountScopedDict()
-_tags = AccountScopedDict()
+_file_systems = AccountRegionScopedDict()
+_mount_targets = AccountRegionScopedDict()
+_access_points = AccountRegionScopedDict()
+_policies = AccountRegionScopedDict()
+_sync_configs = AccountRegionScopedDict()
+_tags = AccountRegionScopedDict()
+
+
+def _clear_state():
+ _file_systems.clear()
+ _mount_targets.clear()
+ _access_points.clear()
+ _policies.clear()
+ _sync_configs.clear()
+ _tags.clear()
def get_state():
@@ -65,12 +75,68 @@ def get_state():
def restore_state(data):
if not data:
return
+ _clear_state()
_file_systems.update(data.get("file_systems", {}))
- _mount_targets.update(data.get("mount_targets", {}))
_access_points.update(data.get("access_points", {}))
- _policies.update(data.get("policies", {}))
- _sync_configs.update(data.get("sync_configs", {}))
- _tags.update(data.get("tags", {}))
+
+ resource_regions = {
+ (account_id, resource_id): region
+ for store in (_file_systems, _access_points)
+ for (account_id, region, resource_id), _resource in store.all_items()
+ }
+ _restore_child_store(
+ _mount_targets,
+ data.get("mount_targets", {}),
+ resource_regions,
+ lambda key, value: value.get("fileSystemId", key),
+ _mount_target_legacy_region,
+ )
+ for store, key in (
+ (_policies, "policies"),
+ (_sync_configs, "sync_configs"),
+ (_tags, "tags"),
+ ):
+ _restore_child_store(
+ store,
+ data.get(key, {}),
+ resource_regions,
+ lambda resource_id, _value: resource_id,
+ )
+
+
+def _mount_target_legacy_region(key, value):
+ availability_zone_id = value.get("availabilityZoneId", "")
+ region, separator, zone_id = availability_zone_id.rpartition("-az")
+ if separator and region and zone_id.isdigit():
+ return region
+ return _mount_targets._region_for_legacy_value(key, value)
+
+
+def _restore_child_store(
+ store, restored, resource_regions, parent_id, legacy_region=None
+):
+ """Adopt legacy child state into its file system or access point region."""
+ if isinstance(restored, AccountRegionScopedDict):
+ store.update(restored)
+ return
+
+ if isinstance(restored, AccountScopedDict):
+ items = restored._data.items()
+ else:
+ account_id = get_account_id()
+ items = (((account_id, key), value) for key, value in restored.items())
+
+ for (account_id, key), value in items:
+ fallback_region = (
+ legacy_region(key, value)
+ if legacy_region is not None
+ else store._region_for_legacy_value(key, value)
+ )
+ region = resource_regions.get(
+ (account_id, parent_id(key, value)),
+ fallback_region,
+ )
+ store.set_scoped(account_id, region, key, value)
try:
@@ -82,12 +148,7 @@ except Exception:
def reset():
- _file_systems.clear()
- _mount_targets.clear()
- _access_points.clear()
- _policies.clear()
- _sync_configs.clear()
- _tags.clear()
+ _clear_state()
# ---------------------------------------------------------------------------
diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index 24e8f007..87acb611 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -1868,6 +1868,45 @@ def test_appsync_region_scoped_state_is_rejected_by_v2_reader(
assert persistence.load_state("appsync") is None
+def test_s3files_region_scoped_state_is_rejected_by_v2_reader(
+ monkeypatch, tmp_path
+):
+ """A rollback binary must reject S3 Files' regional schema instead of
+ accepting it as v2 and silently dropping every regional store."""
+ import json as _json
+
+ from ministack.core.responses import AccountRegionScopedDict
+
+ monkeypatch.setattr(persistence, "PERSIST_STATE", True)
+ monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
+
+ file_systems = AccountRegionScopedDict()
+ file_systems.set_scoped(
+ "000000000000",
+ "us-west-2",
+ "fs-11111111111111111",
+ {
+ "fileSystemId": "fs-11111111111111111",
+ "fileSystemArn": (
+ "arn:aws:s3files:us-west-2:000000000000:"
+ "file-system/fs-11111111111111111"
+ ),
+ },
+ )
+ persistence.save_state("s3files", {"file_systems": file_systems})
+
+ raw = _json.loads((tmp_path / "s3files.json").read_text())
+ assert raw["__ministack_format__"] == 3
+ loaded_file_systems = persistence.load_state("s3files")["file_systems"]
+ assert loaded_file_systems.get_scoped(
+ "000000000000", "us-west-2", "fs-11111111111111111"
+ )["fileSystemId"] == "fs-11111111111111111"
+
+ # Simulate the previous binary, whose highest understood format is v2.
+ monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
+ assert persistence.load_state("s3files") is None
+
+
def test_resource_groups_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
diff --git a/tests/test_s3files.py b/tests/test_s3files.py
index 91433cb8..0fe5d76c 100644
--- a/tests/test_s3files.py
+++ b/tests/test_s3files.py
@@ -17,23 +17,20 @@ BUCKET_ARN = "arn:aws:s3:::test-bucket"
ROLE_ARN = "arn:aws:iam::000000000000:role/s3files-role"
-def _auth(account="000000000000"):
+def _auth(account="000000000000", region="us-east-1"):
return (
f"AWS4-HMAC-SHA256 "
- f"Credential={account}/20260504/us-east-1/s3files/aws4_request, "
+ f"Credential={account}/20260504/{region}/s3files/aws4_request, "
f"SignedHeaders=host, Signature=00"
)
-_AUTH = _auth()
-
-
-def _req(method, path, body=None, query=None, account=None):
+def _req(method, path, body=None, query=None, account=None, region="us-east-1"):
url = ENDPOINT + path
if query:
url += "?" + urllib.parse.urlencode(query, doseq=True)
data = None
- headers = {"Authorization": _auth(account) if account else _AUTH}
+ headers = {"Authorization": _auth(account or "000000000000", region)}
if body is not None:
data = json.dumps(body).encode()
headers["Content-Type"] = "application/json"
@@ -86,6 +83,191 @@ def test_create_file_system_uses_put_and_camel_case():
_req("DELETE", f"/file-systems/{fs_id}")
+def test_file_systems_are_region_scoped():
+ _, east_fs = _req(
+ "PUT",
+ "/file-systems",
+ {"bucket": BUCKET_ARN, "roleArn": ROLE_ARN},
+ region="us-east-1",
+ )
+ _, west_fs = _req(
+ "PUT",
+ "/file-systems",
+ {"bucket": BUCKET_ARN, "roleArn": ROLE_ARN},
+ region="us-west-2",
+ )
+ try:
+ _, east_list = _req("GET", "/file-systems", region="us-east-1")
+ _, west_list = _req("GET", "/file-systems", region="us-west-2")
+ east_ids = {fs["fileSystemId"] for fs in east_list["fileSystems"]}
+ west_ids = {fs["fileSystemId"] for fs in west_list["fileSystems"]}
+
+ assert east_fs["fileSystemId"] in east_ids
+ assert east_fs["fileSystemId"] not in west_ids
+ assert west_fs["fileSystemId"] in west_ids
+ assert west_fs["fileSystemId"] not in east_ids
+ assert ":us-east-1:" in east_fs["fileSystemArn"]
+ assert ":us-west-2:" in west_fs["fileSystemArn"]
+ finally:
+ _req(
+ "DELETE",
+ f"/file-systems/{east_fs['fileSystemId']}",
+ region="us-east-1",
+ )
+ _req(
+ "DELETE",
+ f"/file-systems/{west_fs['fileSystemId']}",
+ region="us-west-2",
+ )
+
+
+def test_legacy_child_state_restores_beside_parent_resource():
+ from ministack.core.responses import (
+ AccountScopedDict,
+ get_account_id,
+ get_region,
+ set_request_account_id,
+ set_request_region,
+ )
+ from ministack.services import s3files as service
+
+ original_account = get_account_id()
+ original_region = get_region()
+ account_id = "111111111111"
+ boot_region = "us-east-1"
+ resource_region = "us-west-2"
+ fs_id = "fs-11111111111111111"
+ mt_id = "fsmt-11111111111111111"
+ ap_id = "fsap-11111111111111111"
+ file_systems = AccountScopedDict()
+ mount_targets = AccountScopedDict()
+ access_points = AccountScopedDict()
+
+ set_request_account_id(account_id)
+ set_request_region(boot_region)
+ file_systems[fs_id] = {
+ "fileSystemId": fs_id,
+ "fileSystemArn": (
+ f"arn:aws:s3files:{resource_region}:{account_id}:file-system/{fs_id}"
+ ),
+ }
+ mount_targets[mt_id] = {"mountTargetId": mt_id, "fileSystemId": fs_id}
+ access_points[ap_id] = {
+ "accessPointId": ap_id,
+ "fileSystemId": fs_id,
+ "accessPointArn": (
+ f"arn:aws:s3files:{resource_region}:{account_id}:"
+ f"file-system/{fs_id}/access-point/{ap_id}"
+ ),
+ }
+ payload = {
+ "file_systems": file_systems,
+ "mount_targets": mount_targets,
+ "access_points": access_points,
+ }
+ for state_key, resource_id, value in (
+ ("policies", fs_id, "legacy-policy"),
+ ("sync_configs", fs_id, {"latestVersionNumber": 1}),
+ ("tags", ap_id, [{"key": "legacy", "value": "true"}]),
+ ):
+ store = AccountScopedDict()
+ store[resource_id] = value
+ payload[state_key] = store
+
+ service.reset()
+ try:
+ service.restore_state(payload)
+ assert service._file_systems.get_scoped(
+ account_id, resource_region, fs_id
+ )["fileSystemId"] == fs_id
+ assert service._mount_targets.get_scoped(
+ account_id, resource_region, mt_id
+ )["fileSystemId"] == fs_id
+ assert service._access_points.get_scoped(
+ account_id, resource_region, ap_id
+ )["fileSystemId"] == fs_id
+ assert service._policies.get_scoped(
+ account_id, resource_region, fs_id
+ ) == "legacy-policy"
+ assert service._sync_configs.get_scoped(
+ account_id, resource_region, fs_id
+ ) == {"latestVersionNumber": 1}
+ assert service._tags.get_scoped(account_id, resource_region, ap_id) == [
+ {"key": "legacy", "value": "true"}
+ ]
+ finally:
+ service.reset()
+ set_request_account_id(original_account)
+ set_request_region(original_region)
+
+
+def test_legacy_orphaned_mount_target_restores_to_availability_zone_region():
+ from ministack.core.responses import (
+ AccountScopedDict,
+ get_account_id,
+ get_region,
+ set_request_account_id,
+ set_request_region,
+ )
+ from ministack.services import s3files as service
+
+ original_account = get_account_id()
+ original_region = get_region()
+ account_id = "111111111111"
+ boot_region = "us-east-1"
+ resource_region = "us-west-2"
+ mt_id = "fsmt-11111111111111111"
+ mount_targets = AccountScopedDict()
+
+ set_request_account_id(account_id)
+ set_request_region(boot_region)
+ mount_targets[mt_id] = {
+ "mountTargetId": mt_id,
+ "fileSystemId": "fs-11111111111111111",
+ "availabilityZoneId": f"{resource_region}-az1",
+ }
+
+ service.reset()
+ try:
+ service.restore_state({"mount_targets": mount_targets})
+ assert service._mount_targets.get_scoped(
+ account_id, resource_region, mt_id
+ )["mountTargetId"] == mt_id
+ assert (
+ service._mount_targets.get_scoped(account_id, boot_region, mt_id) is None
+ )
+ finally:
+ service.reset()
+ set_request_account_id(original_account)
+ set_request_region(original_region)
+
+
+def test_reset_clears_s3files_state_across_regions():
+ from ministack.core.responses import get_region, set_request_region
+ from ministack.services import s3files as service
+
+ original_region = get_region()
+ stores = (
+ service._file_systems,
+ service._mount_targets,
+ service._access_points,
+ service._policies,
+ service._sync_configs,
+ service._tags,
+ )
+ service.reset()
+ try:
+ for region in ("us-east-1", "us-west-2"):
+ set_request_region(region)
+ for store in stores:
+ store[f"resource-{region}"] = {"region": region}
+ service.reset()
+ assert all(not store.has_any() for store in stores)
+ finally:
+ service.reset()
+ set_request_region(original_region)
+
+
def test_create_file_system_post_is_not_supported():
status, body = _req("POST", "/file-systems", {
"bucket": BUCKET_ARN,