Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
"""
Regression tests for the persistence-symmetry architectural bug.
Background
----------
When PERSIST_STATE=1, the registry-derived `_state_map` in
`ministack/app.py` supplies each loaded service's `get_state()` function
to `save_all()` at shutdown. Startup restores saved snapshots centrally
through each service's `load_persisted_state()` method.
These tests protect persistence round trips and the registry contract.
"""
import ast
import importlib
from pathlib import Path
import pytest
from ministack.app import _state_map # noqa: E402 (intentional internal import)
from ministack.core import persistence
def _module(mod_name):
return importlib.import_module(f"ministack.services.{mod_name}")
def _registered_service_modules():
"""Return all modules that participate in the service-state contract.
The optional sub_modules registry field is included before Phase 2 adds
it, so adding one automatically extends this contract test.
"""
from ministack.app import SERVICE_REGISTRY
return sorted(
{
module
for config in SERVICE_REGISTRY.values()
for module in (config["module"], *config.get("sub_modules", ()))
}
)
@pytest.mark.parametrize("mod_name", _registered_service_modules())
def test_registered_service_defines_state_contract(mod_name):
"""Every registered service exposes the uniform persistence API."""
spec = importlib.util.find_spec(f"ministack.services.{mod_name}")
assert spec and spec.origin, f"Cannot find service module {mod_name}"
module_ast = ast.parse(Path(spec.origin).read_text())
functions = {
node.name
for node in module_ast.body
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
}
assert {"get_state", "load_persisted_state", "reset"} <= functions, (
f"Service module {mod_name} must define get_state(), "
"load_persisted_state(data), and reset()."
)
assert "restore_state" not in functions, (
f"Service module {mod_name} exposes restore_state(); use the public "
"load_persisted_state(data) contract instead."
)
def test_tests_restore_state_only_through_the_public_contract():
"""Tests must not couple themselves to service implementation helpers."""
private_calls = []
for path in Path(__file__).parent.glob("test_*.py"):
tree = ast.parse(path.read_text())
for node in ast.walk(tree):
if isinstance(node, ast.Attribute) and node.attr == "_restore_state":
private_calls.append(f"{path.name}:{node.lineno}")
elif isinstance(node, ast.Name) and node.id == "_restore_state":
private_calls.append(f"{path.name}:{node.lineno}")
assert not private_calls, (
"Tests must restore state through load_persisted_state(), not private "
f"helpers: {', '.join(private_calls)}"
)
@pytest.mark.parametrize("svc_key,mod_name", sorted(_state_map.items()))
def test_service_has_restore_path(svc_key, mod_name):
"""Every service in `_state_map` must expose a way to restore its own state.
The registry-driven app loader dispatches each saved snapshot through the
module's `load_persisted_state(data)` contract. Without that method its
state is written on shutdown and dropped on the next boot, silently.
"""
mod = _module(mod_name)
if not mod.get_state():
pytest.skip(f"{mod_name} holds no state to restore")
assert callable(getattr(mod, "load_persisted_state", None)), (
f"Service `{svc_key}` (module `{mod_name}`) is in `_state_map` and will "
f"be saved on shutdown, but has no restore path on startup."
)
def test_account_region_scoped_dict_isolates_account_and_region():
"""Account+region scoped stores keep same-name resources independent."""
from ministack.core.responses import (
AccountRegionScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
original_account = get_account_id()
original_region = get_region()
try:
store = AccountRegionScopedDict()
set_request_account_id("111111111111")
set_request_region("us-east-1")
store["same-name"] = {"scope": "111/east"}
set_request_region("us-west-2")
store["same-name"] = {"scope": "111/west"}
set_request_account_id("222222222222")
set_request_region("us-east-1")
store["same-name"] = {"scope": "222/east"}
assert len(store) == 1
assert store["same-name"] == {"scope": "222/east"}
set_request_account_id("111111111111")
assert store["same-name"] == {"scope": "111/east"}
set_request_region("us-west-2")
assert store["same-name"] == {"scope": "111/west"}
finally:
set_request_account_id(original_account)
set_request_region(original_region)
def test_account_region_scoped_dict_persistence_round_trip(monkeypatch, tmp_path):
"""Account+region scoped stores must survive the JSON persistence path."""
from ministack.core.responses import (
AccountRegionScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
original_account = get_account_id()
original_region = get_region()
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
try:
store = AccountRegionScopedDict()
set_request_account_id("111111111111")
set_request_region("us-east-1")
store["same-name"] = {"region": "us-east-1"}
store[("compound", "key")] = {"region": "us-east-1"}
set_request_region("us-west-2")
store["same-name"] = {"region": "us-west-2"}
persistence.save_state("account-region-scoped", {"store": store})
loaded = persistence.load_state("account-region-scoped")
assert loaded is not None
restored = loaded["store"]
set_request_region("us-east-1")
assert restored["same-name"] == {"region": "us-east-1"}
assert restored[("compound", "key")] == {"region": "us-east-1"}
set_request_region("us-west-2")
assert restored["same-name"] == {"region": "us-west-2"}
finally:
set_request_account_id(original_account)
set_request_region(original_region)
def test_account_region_scoped_dict_adopts_legacy_values_to_arn_region():
"""Legacy account-scoped stores should migrate into the region in the value ARN."""
from ministack.core.responses import (
AccountRegionScopedDict,
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
original_account = get_account_id()
original_region = get_region()
try:
legacy = AccountScopedDict()
legacy._data[("111111111111", "west-sm")] = {
"stateMachineArn": "arn:aws:states:us-west-2:111111111111:stateMachine:west-sm",
}
legacy._data[("111111111111", "no-arn")] = {"name": "no-arn"}
restored = AccountRegionScopedDict()
set_request_account_id("111111111111")
set_request_region("us-east-1")
restored.update(legacy)
assert restored.get_scoped("111111111111", "us-west-2", "west-sm") == {
"stateMachineArn": "arn:aws:states:us-west-2:111111111111:stateMachine:west-sm",
}
assert restored.get_scoped("111111111111", "us-east-1", "west-sm") is None
assert restored.get_scoped("111111111111", "us-east-1", "no-arn") == {"name": "no-arn"}
finally:
set_request_account_id(original_account)
set_request_region(original_region)
def test_account_region_scoped_dict_update_preserves_tuple_resource_keys():
"""Plain dict updates should treat tuple keys as normal resource keys."""
from ministack.core.responses import (
AccountRegionScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
original_account = get_account_id()
original_region = get_region()
try:
store = AccountRegionScopedDict()
set_request_account_id("111111111111")
set_request_region("us-east-1")
store.update({("bucket", "object-key"): {"value": "east"}})
assert store[("bucket", "object-key")] == {"value": "east"}
assert store.get_scoped("111111111111", "us-east-1", ("bucket", "object-key")) == {"value": "east"}
assert store.get_scoped("bucket", "us-east-1", "object-key") is None
finally:
set_request_account_id(original_account)
set_request_region(original_region)
def test_account_region_scoped_dict_update_adopts_plain_values_to_arn_region():
"""Plain legacy dict restores should use the resource ARN region when present."""
from ministack.core.responses import (
AccountRegionScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
original_account = get_account_id()
original_region = get_region()
try:
store = AccountRegionScopedDict()
set_request_account_id("111111111111")
set_request_region("us-east-1")
store.update({
"west-sm": {
"stateMachineArn": "arn:aws:states:us-west-2:111111111111:stateMachine:west-sm",
},
})
assert store.get_scoped("111111111111", "us-west-2", "west-sm") == {
"stateMachineArn": "arn:aws:states:us-west-2:111111111111:stateMachine:west-sm",
}
assert store.get_scoped("111111111111", "us-east-1", "west-sm") is None
finally:
set_request_account_id(original_account)
set_request_region(original_region)
def test_account_region_scoped_dict_update_prefers_key_arn_region():
"""Legacy ARN-keyed stores should not be scoped by unrelated value ARNs."""
from ministack.core.responses import (
AccountRegionScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
original_account = get_account_id()
original_region = get_region()
east_arn = "arn:aws:states:us-east-1:111111111111:stateMachine:east-sm"
west_role_arn = "arn:aws:iam:us-west-2:111111111111:role/west-role"
try:
store = AccountRegionScopedDict()
set_request_account_id("111111111111")
set_request_region("us-west-2")
store.update({east_arn: {"RoleArn": west_role_arn}})
assert store.get_scoped("111111111111", "us-east-1", east_arn) == {"RoleArn": west_role_arn}
assert store.get_scoped("111111111111", "us-west-2", east_arn) is None
finally:
set_request_account_id(original_account)
set_request_region(original_region)
def test_account_scoped_dict_helpers_remain_account_only():
"""Scoped helper methods on AccountScopedDict continue to ignore region."""
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
set_request_account_id,
)
original_account = get_account_id()
try:
store = AccountScopedDict()
store.set_scoped("111111111111", "us-west-2", "same-name", {"value": "helper"})
set_request_account_id("111111111111")
assert store["same-name"] == {"value": "helper"}
store["normal"] = {"value": "normal"}
assert store.get_scoped("111111111111", "us-east-1", "normal") == {"value": "normal"}
assert store.contains_scoped("111111111111", "us-west-2", "normal")
assert store.pop_scoped("111111111111", "us-west-2", "normal") == {"value": "normal"}
assert "normal" not in store
finally:
set_request_account_id(original_account)
def test_appsync_events_legacy_children_follow_parent_api_region():
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import appsync_events as service
original_account = get_account_id()
original_region = get_region()
account_id = "111111111111"
boot_region = "us-east-1"
api_region = "us-west-2"
api_id = "legacy-regional-api"
orphan_api_id = "legacy-orphan-api"
apis = AccountScopedDict()
apis._data[(account_id, api_id)] = {
"name": "legacy-api",
"apiArn": f"arn:aws:appsync:{api_region}:{account_id}:apis/{api_id}",
}
namespaces = AccountScopedDict()
namespaces._data[(account_id, api_id)] = {
"default": {"created": 1},
}
namespaces._data[(account_id, orphan_api_id)] = {
"orphan": {"created": 2},
}
api_keys = AccountScopedDict()
api_keys._data[(account_id, api_id)] = {
"da2-legacy": {"description": "legacy"},
}
service.reset()
try:
set_request_account_id(account_id)
set_request_region(boot_region)
service.load_persisted_state(
{
"apis": apis,
"channel_namespaces": namespaces,
"api_keys": api_keys,
}
)
assert service._apis.get_scoped(account_id, api_region, api_id)[
"apiArn"
].startswith(f"arn:aws:appsync:{api_region}:")
assert service._channel_namespaces.get_scoped(
account_id, api_region, api_id
) == {"default": {"created": 1}}
assert service._api_keys.get_scoped(account_id, api_region, api_id) == {
"da2-legacy": {"description": "legacy"},
}
assert (
service._channel_namespaces.get_scoped(
account_id, boot_region, api_id
)
is None
)
assert (
service._api_keys.get_scoped(account_id, boot_region, api_id) is None
)
assert service._channel_namespaces.get_scoped(
account_id, boot_region, orphan_api_id
) == {"orphan": {"created": 2}}
finally:
service.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_appsync_events_v2_region_scoped_state_round_trip(monkeypatch, tmp_path):
import json as _json
from ministack.core.responses import (
AccountRegionScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import appsync_events as service
original_account = get_account_id()
original_region = get_region()
account_id = "111111111111"
boot_region = "us-east-1"
api_region = "us-west-2"
api_id = "current-regional-api"
apis = AccountRegionScopedDict()
apis.set_scoped(
account_id,
api_region,
api_id,
{
"name": "current-api",
"apiArn": f"arn:aws:appsync:{api_region}:{account_id}:apis/{api_id}",
},
)
namespaces = AccountRegionScopedDict()
namespaces.set_scoped(
account_id,
api_region,
api_id,
{"default": {"created": 1}},
)
api_keys = AccountRegionScopedDict()
api_keys.set_scoped(
account_id,
api_region,
api_id,
{"da2-current": {"description": "current"}},
)
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
(tmp_path / "appsync_events.json").write_text(
_json.dumps(
{
"__ministack_format__": 2,
"payload": {
"apis": apis,
"channel_namespaces": namespaces,
"api_keys": api_keys,
},
},
default=persistence._json_default,
)
)
loaded = persistence.load_state("appsync_events")
service.reset()
try:
set_request_account_id(account_id)
set_request_region(boot_region)
service.load_persisted_state(loaded)
assert service._apis.get_scoped(account_id, api_region, api_id)[
"apiArn"
].startswith(f"arn:aws:appsync:{api_region}:")
assert service._channel_namespaces.get_scoped(
account_id, api_region, api_id
) == {"default": {"created": 1}}
assert service._api_keys.get_scoped(account_id, api_region, api_id) == {
"da2-current": {"description": "current"},
}
assert service._apis.get_scoped(account_id, boot_region, api_id) is None
finally:
service.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_pipes_is_in_state_map():
"""The registry-derived map includes the Pipes service contract."""
pipes = _module("pipes")
assert hasattr(pipes, "get_state"), "pipes module no longer has get_state — update this test"
assert "pipes" in _state_map, (
"`pipes` defines get_state() but is missing from `_state_map` in "
"app.py — its state is never saved on shutdown."
)
def test_state_map_is_derived_from_service_registry():
"""The registry is the only declaration point for persisted modules."""
from ministack.app import SERVICE_REGISTRY, _registry_module_names, _registry_state_map
expected = {}
for config in SERVICE_REGISTRY.values():
module = config["module"]
expected[config.get("state_key", module)] = module
expected.update({sub_module: sub_module for sub_module in config.get("sub_modules", ())})
assert _state_map == expected
assert _registry_state_map() == expected
assert _registry_module_names() == set(expected.values())
def test_save_dict_includes_sibling_imported_modules():
"""Regression for #704. `appsync_events` is reached only via sibling
import from `appsync.py` for REST traffic — the routed handler for
`appsync-events` never fires (Event API traffic arrives under the
`appsync` credential scope). Same shape: `apigateway_v1` is reached
via sibling import from `apigateway.py`. If the shutdown save loop
only consults `_loaded_modules` (populated by `_get_module`), those
sibling-imported modules are silently skipped and their state is
dropped. The fallback through `sys.modules` is the fix."""
import sys as _sys
# Force-import appsync_events the way appsync.py does it — a plain
# sibling import that bypasses `_get_module` and therefore does NOT
# populate `_loaded_modules`.
import ministack.services.appsync_events # noqa: F401
from ministack.app import _build_persistence_save_dict, _loaded_modules
# Simulate the bug condition: module is in sys.modules but absent
# from _loaded_modules.
saved = _loaded_modules.pop("appsync_events", None)
try:
assert "ministack.services.appsync_events" in _sys.modules, (
"test premise broken — module isn't in sys.modules"
)
assert "appsync_events" not in _loaded_modules, (
"test premise broken — module is still in _loaded_modules"
)
save_dict = _build_persistence_save_dict()
assert "appsync_events" in save_dict, (
"shutdown save loop dropped appsync_events even though the "
"module was imported via a sibling import. The sys.modules "
"fallback in `_build_persistence_save_dict` is missing or broken."
)
# The value must be the bound get_state method, not the result.
assert callable(save_dict["appsync_events"]), (
"save_dict should map to a callable (get_state method ref) — "
"save_all invokes it. Got %r" % (save_dict["appsync_events"],)
)
finally:
if saved is not None:
_loaded_modules["appsync_events"] = saved
def test_save_dict_skips_modules_never_imported(monkeypatch):
"""The sys.modules fallback must NOT save state for modules that
were never imported at all — there's no state to capture and any
`get_state()` call on a non-imported module would attribute-error.
Defensive guard: ensure the fallback path's `hasattr` check works."""
import sys
from ministack.app import _build_persistence_save_dict, _loaded_modules, _state_map
# Every registered module is now represented in `_state_map`, so force a
# known entry absent from both sources instead of relying on an exclusion.
target = "account"
assert target in _state_map.values(), "test premise broken — account must be registry-derived"
monkeypatch.delitem(_loaded_modules, target, raising=False)
monkeypatch.delitem(sys.modules, f"ministack.services.{target}", raising=False)
# Even after the fallback path runs, an unloaded module must not appear.
save_dict = _build_persistence_save_dict()
assert target not in save_dict, (
"save_dict picked up a module that was absent from both module caches — "
"the fallback's unloaded-module guard is broken."
)
# ── Functional round-trip tests ────────────────────────────────────────
def _round_trip(mod_name, svc_key, populate_fn, observe_fn):
"""Helper: populate -> save -> reset -> registry-style restore -> observe."""
mod = _module(mod_name)
mod.reset()
populate_fn(mod)
snapshot = mod.get_state()
# Persist via the same code path as `save_all` would use.
persistence.save_state(svc_key, snapshot)
# Wipe in-memory state — this simulates a process restart.
mod.reset()
# Restore via the same uniform contract used by the registry loader.
loaded = persistence.load_state(svc_key)
assert loaded is not None, (
f"persistence.load_state({svc_key!r}) returned None — state file "
"was not written by save_state(). Check `_state_map` membership "
"and `get_state()` correctness."
)
mod.load_persisted_state(loaded)
# Cleanup state file before observation, so a failure doesn't pollute
# the next test run.
import os
state_file = os.path.join(persistence.STATE_DIR, f"{svc_key}.json")
if os.path.exists(state_file):
os.remove(state_file)
observe_fn(mod)
mod.reset()
@pytest.fixture(autouse=True)
def _enable_persistence(monkeypatch, tmp_path):
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
def test_autoscaling_round_trip():
def populate(mod):
# Drive the state via the module's own dict directly — minimal
# surface, no SDK needed.
mod._launch_configs["lc-test"] = {"LaunchConfigurationName": "lc-test"}
mod._asgs["asg-test"] = {"AutoScalingGroupName": "asg-test", "MinSize": 1}
def observe(mod):
assert "lc-test" in mod._launch_configs
assert "asg-test" in mod._asgs
_round_trip("autoscaling", "autoscaling", populate, observe)
def test_backup_round_trip():
def populate(mod):
mod._vaults["vault-test"] = {"BackupVaultName": "vault-test"}
def observe(mod):
assert "vault-test" in mod._vaults
_round_trip("backup", "backup", populate, observe)
def test_route53_private_zone_vpcs_round_trip():
vpc = {"VPCRegion": "us-west-2", "VPCId": "vpc-0123456789abcdef0"}
body = (
'<CreateHostedZoneRequest xmlns="https://route53.amazonaws.com/doc/2013-04-01/">'
"<Name>persisted.example.internal</Name><CallerReference>persist-1</CallerReference>"
f"<VPC><VPCRegion>{vpc['VPCRegion']}</VPCRegion><VPCId>{vpc['VPCId']}</VPCId></VPC>"
"</CreateHostedZoneRequest>"
).encode()
zone_ids = []
def populate(mod):
status, _, _ = mod._create_hosted_zone(body, {})
assert status == 201
zone_ids.extend(mod._zones.keys())
# A zone persisted before VPC associations existed has no "vpcs" key.
mod._zones["ZLEGACYPRIVATE1"] = {
"id": "ZLEGACYPRIVATE1", "name": "legacy.example.internal.",
"caller_reference": "legacy", "comment": "", "private": True,
}
def observe(mod):
assert mod._zones[zone_ids[0]]["vpcs"] == [vpc]
status, _, out = mod._list_hosted_zones_by_vpc({"vpcid": vpc["VPCId"], "vpcregion": vpc["VPCRegion"]})
assert status == 200 and zone_ids[0].encode() in out
status, _, out = mod._get_hosted_zone("ZLEGACYPRIVATE1")
assert status == 200 and b"<VPCs />" in out
_round_trip("route53", "route53", populate, observe)
def test_s3_mrap_round_trip():
"""A multi-region access point outlives a restart. Its alias is baked into
client configuration, so member buckets coming back without the alias
fronting them fails where a missing bucket would not."""
def populate(mod):
mod._mraps["ea0672af90364.mrap"] = {
"Name": "app-release",
"Alias": "ea0672af90364.mrap",
"Regions": ["app-us-east-1-release", "app-eu-west-1-release"],
"CreatedAt": "2026-01-01T00:00:00Z",
}
def observe(mod):
mrap = mod._mraps["ea0672af90364.mrap"]
assert mrap["Name"] == "app-release"
assert mrap["Regions"] == ["app-us-east-1-release", "app-eu-west-1-release"]
_round_trip("s3", "s3", populate, observe)
def test_s3_access_points_round_trip_every_tenant():
"""Access points are scoped by account and region; a restart keeps all of
them, not only the tenant that saved."""
record = {"Name": "ap", "Bucket": "data", "Alias": "ap-abc-s3alias", "NetworkOrigin": "Internet"}
def populate(mod):
mod._access_points.set_scoped("111111111111", "us-east-1", "ap", dict(record))
mod._access_points.set_scoped("222222222222", "eu-west-1", "ap", {**record, "Bucket": "other"})
def observe(mod):
assert mod._access_points.get_scoped("111111111111", "us-east-1", "ap")["Bucket"] == "data"
assert mod._access_points.get_scoped("222222222222", "eu-west-1", "ap")["Bucket"] == "other"
_round_trip("s3", "s3", populate, observe)
def test_s3_object_lambda_access_points_round_trip_without_pending_requests():
"""Object Lambda Access Points outlive a restart; requests waiting for a
WriteGetObjectResponse are in flight only and are never saved."""
def populate(mod):
mod._object_lambda_access_points.set_scoped("111111111111", "us-east-1", "olap", {
"Name": "olap", "Alias": {"Value": "olap-abc--ol-s3", "Status": "READY"}})
mod._olap_requests[("route", "token")] = {"done": False}
def observe(mod):
assert mod._object_lambda_access_points.get_scoped(
"111111111111", "us-east-1", "olap")["Alias"]["Value"] == "olap-abc--ol-s3"
assert "olap_requests" not in mod.get_state()
_round_trip("s3", "s3", populate, observe)
def test_cloudfront_vpc_origins_round_trip_every_account():
"""CloudFront VPC origins are account-scoped; a restart keeps every account's."""
def record(account, vo_id, version):
return {"Id": vo_id, "Arn": f"arn:aws:cloudfront::{account}:vpcorigin/{vo_id}", "AccountId": account,
"Status": "Deployed", "Version": version, "VpcOriginEndpointConfig": {"Name": "api"}}
def populate(mod):
mod._vpc_origins.set_scoped("111111111111", "us-east-1", "vo_a", record("111111111111", "vo_a", 4))
mod._vpc_origins.set_scoped("222222222222", "us-east-1", "vo_b", record("222222222222", "vo_b", 6))
# A deleted origin had reached version 8: its ETags must stay valid too (R1 U10/delete-C-after-gone).
for version in range(1, 9):
mod._vpc_origin_etag(version)
def observe(mod):
assert mod._vpc_origins.get_scoped("111111111111", "us-east-1", "vo_a")["AccountId"] == "111111111111"
assert mod._vpc_origins.get_scoped("222222222222", "us-east-1", "vo_b")["Version"] == 6
assert not mod._vpc_origins.contains_scoped("111111111111", "us-east-1", "vo_b")
# Every version's ETag issued before the restart is valid again for If-Match.
assert sorted(set(mod._vpc_origin_etags_issued.values())) == list(range(1, 9))
_round_trip("cloudfront", "cloudfront", populate, observe)
def test_transcribe_round_trip():
import time as _time
def populate(mod):
mod._jobs["job-test"] = {
"TranscriptionJobName": "job-test",
"TranscriptionJobStatus": "COMPLETED",
"LanguageCode": "en-US",
"Media": {"MediaFileUri": "s3://media/call.mp3"},
"Transcript": {
"TranscriptFileUri": "http://localhost:4566/out/job-test.json"
},
"CreationTime": _time.time(),
"StartTime": _time.time(),
"CompletionTime": _time.time(),
"OutputLocationType": "SERVICE_BUCKET",
"_run_id": "run-1",
"_output_bucket": "out",
"_output_key": "job-test.json",
}
def observe(mod):
job = mod._jobs.get("job-test")
assert job is not None
assert job["TranscriptionJobStatus"] == "COMPLETED"
# A completed job's transcript pointer and media reference are what
# callers read after a warm boot; losing either leaves GetTranscriptionJob
# reporting COMPLETED with nothing to fetch.
assert job["Transcript"]["TranscriptFileUri"].endswith("job-test.json")
assert job["Media"]["MediaFileUri"] == "s3://media/call.mp3"
assert job["_output_bucket"] == "out"
_round_trip("transcribe", "transcribe", populate, observe)
def test_transcribe_jobs_round_trip_outside_boot_region():
"""Transcribe jobs are region-scoped. `AccountRegionScopedDict.__bool__`
is scope-relative, so a snapshot holding jobs only in regions other than
the one restore runs in must not be treated as empty."""
import time as _time
from ministack.core.responses import request_scope
def populate(mod):
with request_scope("000000000000", "eu-west-1"):
mod._jobs["eu-job"] = {
"TranscriptionJobName": "eu-job",
"TranscriptionJobStatus": "COMPLETED",
"CreationTime": _time.time(),
"CompletionTime": _time.time(),
"Media": {"MediaFileUri": "s3://b/a.mp3"},
"OutputLocationType": "SERVICE_BUCKET",
"_output_bucket": "b",
"_output_key": "k.json",
}
def observe(mod):
with request_scope("000000000000", "eu-west-1"):
job = mod._jobs.get("eu-job")
assert job is not None, "a job outside the restoring region was dropped"
assert job["TranscriptionJobStatus"] == "COMPLETED"
_round_trip("transcribe", "transcribe", populate, observe)
def test_transcribe_round_trip_fails_a_job_left_mid_flight():
"""A restart leaves no worker behind, so a job restored as QUEUED or
IN_PROGRESS would strand every caller polling GetTranscriptionJob."""
import time as _time
def populate(mod):
mod._jobs["stuck-job"] = {
"TranscriptionJobName": "stuck-job",
"TranscriptionJobStatus": "IN_PROGRESS",
"CreationTime": _time.time(),
"Media": {"MediaFileUri": "s3://b/a.mp3"},
"OutputLocationType": "SERVICE_BUCKET",
"_output_bucket": "b",
"_output_key": "k.json",
}
def observe(mod):
job = mod._jobs.get("stuck-job")
assert job is not None
assert job["TranscriptionJobStatus"] == "FAILED"
assert job["CompletionTime"] is not None
assert "restart" in job["FailureReason"]
_round_trip("transcribe", "transcribe", populate, observe)
def test_translate_round_trip():
import time as _time
def populate(mod):
mod._jobs["1c1838f470806ab9c3e0057f14717bed"] = {
"JobId": "1c1838f470806ab9c3e0057f14717bed",
"JobName": "nightly-transcript-translation",
"JobStatus": "COMPLETED",
"JobDetails": {
"TranslatedDocumentsCount": 2,
"DocumentsWithErrorsCount": 0,
"InputDocumentsCount": 2,
},
"SourceLanguageCode": "en",
"TargetLanguageCodes": ["fr"],
"SubmittedTime": _time.time(),
"EndTime": _time.time(),
"InputDataConfig": {
"S3Uri": "s3://corpus/input/",
"ContentType": "application/x-xliff+xml",
},
"OutputDataConfig": {
"S3Uri": (
"s3://corpus/output/000000000000-TranslateText-"
"1c1838f470806ab9c3e0057f14717bed/"
)
},
"DataAccessRoleArn": "arn:aws:iam::000000000000:role/TranslateBatchRole",
"_run_id": "run-1",
"_client_token": "token-1",
"_output_bucket": "corpus",
"_output_prefix": "output/000000000000-TranslateText-1c1838f470806ab9c3e0057f14717bed/",
}
mod._client_tokens["token-1"] = "1c1838f470806ab9c3e0057f14717bed"
def observe(mod):
job = mod._jobs.get("1c1838f470806ab9c3e0057f14717bed")
assert job is not None
assert job["JobStatus"] == "COMPLETED"
# The rewritten output location is the only pointer a caller has to the
# translated documents; losing it leaves Describe reporting COMPLETED
# with nothing to fetch.
assert job["OutputDataConfig"]["S3Uri"].endswith(
"000000000000-TranslateText-1c1838f470806ab9c3e0057f14717bed/"
)
assert job["JobDetails"]["TranslatedDocumentsCount"] == 2
# The idempotency map has to survive too, or a client retrying with the
# same token after a restart starts a duplicate job.
assert mod._client_tokens.get("token-1") == "1c1838f470806ab9c3e0057f14717bed"
_round_trip("translate", "translate", populate, observe)
def test_translate_jobs_round_trip_outside_boot_region():
"""Translate jobs are region-scoped. `AccountRegionScopedDict.__bool__` is
scope-relative, so a snapshot holding jobs only in regions other than the
one restore runs in must not be treated as empty."""
import time as _time
from ministack.core.responses import request_scope
def populate(mod):
with request_scope("000000000000", "eu-west-1"):
mod._jobs["eu-job"] = {
"JobId": "eu-job",
"JobStatus": "COMPLETED",
"SourceLanguageCode": "en",
"TargetLanguageCodes": ["de"],
"SubmittedTime": _time.time(),
"EndTime": _time.time(),
"InputDataConfig": {"S3Uri": "s3://b/in/", "ContentType": "text/plain"},
"OutputDataConfig": {"S3Uri": "s3://b/out/"},
"_output_bucket": "b",
"_output_prefix": "out/",
}
def observe(mod):
with request_scope("000000000000", "eu-west-1"):
job = mod._jobs.get("eu-job")
assert job is not None, "a job outside the restoring region was dropped"
assert job["JobStatus"] == "COMPLETED"
_round_trip("translate", "translate", populate, observe)
def test_translate_round_trip_fails_a_job_left_mid_flight():
"""A restart leaves no worker behind, so a job restored as SUBMITTED,
IN_PROGRESS or STOP_REQUESTED would strand every caller polling
DescribeTextTranslationJob."""
import time as _time
def populate(mod):
mod._jobs["stuck-job"] = {
"JobId": "stuck-job",
"JobStatus": "IN_PROGRESS",
"SourceLanguageCode": "en",
"TargetLanguageCodes": ["fr"],
"SubmittedTime": _time.time(),
"InputDataConfig": {"S3Uri": "s3://b/in/", "ContentType": "text/plain"},
"OutputDataConfig": {"S3Uri": "s3://b/out/"},
"_output_bucket": "b",
"_output_prefix": "out/",
}
def observe(mod):
job = mod._jobs.get("stuck-job")
assert job is not None
assert job["JobStatus"] == "FAILED"
assert job["EndTime"] is not None
assert "restart" in job["Message"]
_round_trip("translate", "translate", populate, observe)
def test_location_round_trip():
def populate(mod):
mod._trackers["trk-test"] = {
"TrackerName": "trk-test",
"TrackerArn": "arn:aws:geo:us-east-1:000000000000:tracker/trk-test",
"CreateTime": 1000.0,
"UpdateTime": 1000.0,
"positions": {
"dev-1": {
"latest": {"DeviceId": "dev-1", "SampleTime": 1000.0,
"ReceivedTime": 1000.5, "Position": [11.0, 48.0]},
"history": [{"DeviceId": "dev-1", "SampleTime": 1000.0,
"ReceivedTime": 1000.5, "Position": [11.0, 48.0]}],
},
},
}
def observe(mod):
rec = mod._trackers["trk-test"]
assert rec["TrackerArn"].endswith(":tracker/trk-test")
assert rec["positions"]["dev-1"]["latest"]["Position"] == [11.0, 48.0]
_round_trip("location", "location", populate, observe)
def test_cloudformation_round_trip():
"""CloudFormation stack metadata (stacks, events, exports, change sets)
survives a PERSIST_STATE stop/restore cycle — otherwise ListStacks /
DescribeStacks / ListExports come back empty after a warm boot even though
the provisioned resources persist. (#1345, item 8)"""
def populate(mod):
mod._stacks["stk-rt"] = {
"StackName": "stk-rt",
"StackId": "arn:aws:cloudformation:us-east-1:000000000000:stack/stk-rt/rt",
"StackStatus": "CREATE_COMPLETE",
"Outputs": [{"OutputKey": "K", "OutputValue": "V"}],
}
mod._stack_events["arn:aws:cloudformation:us-east-1:000000000000:stack/stk-rt/rt"] = [
{"ResourceStatus": "CREATE_COMPLETE"},
]
mod._exports["exp-rt"] = {"Name": "exp-rt", "Value": "V", "ExportingStackId": "stk-rt"}
mod._change_sets["cs-rt"] = {"ChangeSetName": "cs-rt", "Status": "CREATE_COMPLETE"}
def observe(mod):
stack = mod._stacks.get("stk-rt")
assert stack is not None and stack["StackStatus"] == "CREATE_COMPLETE"
assert stack["Outputs"] == [{"OutputKey": "K", "OutputValue": "V"}]
assert mod._stack_events.get(
"arn:aws:cloudformation:us-east-1:000000000000:stack/stk-rt/rt") is not None
assert mod._exports.get("exp-rt") is not None
assert mod._change_sets.get("cs-rt") is not None
_round_trip("cloudformation", "cloudformation", populate, observe)
def test_cloudformation_drift_and_failed_operation_round_trip():
"""Drift results and the failed operation RollbackStack needs live on the
stack record, so they survive a PERSIST_STATE stop/restore cycle."""
stack_id = "arn:aws:cloudformation:us-east-1:000000000000:stack/stk-drift/rt"
def populate(mod):
drift = {"StackId": stack_id, "LogicalResourceId": "Q", "ResourceType": "AWS::SQS::Queue",
"StackResourceDriftStatus": "MODIFIED", "Timestamp": "2026-09-30T00:00:00.000Z",
"PropertyDifferences": [{"PropertyPath": "/VisibilityTimeout",
"ExpectedValue": "30", "ActualValue": "60",
"DifferenceType": "NOT_EQUAL"}]}
mod._stacks["stk-drift"] = {
"StackName": "stk-drift",
"StackId": stack_id,
"StackStatus": "UPDATE_FAILED",
"DriftInformation": {"StackDriftStatus": "DRIFTED",
"LastCheckTimestamp": "2026-09-30T00:00:00.000Z"},
"_drift_detections": {"det-1": {"StackId": stack_id,
"StackDriftDetectionId": "det-1",
"StackDriftStatus": "DRIFTED",
"DetectionStatus": "DETECTION_COMPLETE",
"DriftedStackResourceCount": 1,
"Timestamp": "2026-09-30T00:00:00.000Z"}},
"_resources": {"Q": {"ResourceType": "AWS::SQS::Queue", "_drift": drift,
"DriftInformation": {"StackResourceDriftStatus": "MODIFIED"}}},
"_failed_operation": {"is_update": True, "created": ["Q"],
"failed_update": ["Q", "AWS::SQS::Queue", "url", None, None, None],
"replaced_ids": [], "previous_stack": {"_resources": {}}},
}
def observe(mod):
stack = mod._stacks.get("stk-drift")
assert stack["DriftInformation"]["StackDriftStatus"] == "DRIFTED"
assert stack["_drift_detections"]["det-1"]["DriftedStackResourceCount"] == 1
assert stack["_resources"]["Q"]["_drift"]["PropertyDifferences"][0]["ActualValue"] == "60"
assert stack["_failed_operation"]["failed_update"][0] == "Q"
assert stack["_failed_operation"]["created"] == ["Q"]
_round_trip("cloudformation", "cloudformation", populate, observe)
def test_backup_region_scoped_v3_state_is_idempotent(monkeypatch, tmp_path):
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
account_id = "123456789012"
region = "us-west-2"
vault_name = "regional-vault"
plan_id = "regional-plan"
selection_id = "regional-selection"
job_id = "regional-job"
def regional_store(key, value):
store = AccountRegionScopedDict()
store.set_scoped(account_id, region, key, value)
return store
state = {
"vaults": regional_store(
vault_name,
{
"BackupVaultName": vault_name,
"BackupVaultArn": f"arn:aws:backup:{region}:{account_id}:backup-vault:{vault_name}",
},
),
"plans": regional_store(
plan_id,
{
"BackupPlanId": plan_id,
"BackupPlanArn": f"arn:aws:backup:{region}:{account_id}:backup-plan:{plan_id}",
},
),
"selections": regional_store(
selection_id,
{
"SelectionId": selection_id,
"BackupPlanId": plan_id,
"Resources": [f"arn:aws:dynamodb:{region}:{account_id}:table/MyTable"],
},
),
"jobs": regional_store(
job_id,
{
"BackupJobId": job_id,
"BackupJobArn": f"arn:aws:backup:{region}:{account_id}:backup-job:{job_id}",
},
),
}
persistence.save_state("backup", state)
first_snapshot = _json.loads((tmp_path / "backup.json").read_text())
assert first_snapshot["__ministack_format__"] == 3
loaded = persistence.load_state("backup")
assert loaded is not None
persistence.save_state("backup", loaded)
second_snapshot = _json.loads((tmp_path / "backup.json").read_text())
assert second_snapshot == first_snapshot
def test_backup_legacy_selection_migration_colocates_with_parent_plan():
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import backup
original_account = get_account_id()
original_region = get_region()
account_id = "123456789012"
boot_region = "us-east-1"
plan_region = "us-west-2"
plan_id = "legacy-plan"
colocated_selection_id = "legacy-selection"
orphan_selection_id = "orphan-selection"
vault_name = "legacy-vault"
job_id = "legacy-job"
def scoped_store(key, value):
store = AccountScopedDict()
store._data[(account_id, key)] = value
return store
try:
set_request_account_id(account_id)
set_request_region(boot_region)
backup.reset()
legacy_state = {
"vaults": scoped_store(
vault_name,
{
"BackupVaultName": vault_name,
"BackupVaultArn": f"arn:aws:backup:{plan_region}:{account_id}:backup-vault:{vault_name}",
},
),
"plans": scoped_store(
plan_id,
{
"BackupPlanId": plan_id,
"BackupPlanArn": f"arn:aws:backup:{plan_region}:{account_id}:backup-plan:{plan_id}",
},
),
"selections": AccountScopedDict(),
"jobs": scoped_store(
job_id,
{
"BackupJobId": job_id,
"BackupJobArn": f"arn:aws:backup:{plan_region}:{account_id}:backup-job:{job_id}",
},
),
}
legacy_state["selections"]._data[(account_id, colocated_selection_id)] = {
"SelectionId": colocated_selection_id,
"BackupPlanId": plan_id,
"Resources": [f"arn:aws:dynamodb:{boot_region}:{account_id}:table/EastTable"],
}
legacy_state["selections"]._data[(account_id, orphan_selection_id)] = {
"SelectionId": orphan_selection_id,
"BackupPlanId": "missing-plan",
"Resources": [f"arn:aws:dynamodb:{plan_region}:{account_id}:table/WestTable"],
}
backup.load_persisted_state(legacy_state)
assert backup._vaults.get_scoped(account_id, plan_region, vault_name) is not None
assert backup._plans.get_scoped(account_id, plan_region, plan_id) is not None
assert backup._jobs.get_scoped(account_id, plan_region, job_id) is not None
assert backup._selections.get_scoped(
account_id, plan_region, colocated_selection_id
)["BackupPlanId"] == plan_id
assert backup._selections.get_scoped(
account_id, boot_region, colocated_selection_id
) is None
assert backup._selections.get_scoped(
account_id, boot_region, orphan_selection_id
)["BackupPlanId"] == "missing-plan"
assert backup._selections.get_scoped(
account_id, plan_region, orphan_selection_id
) is None
finally:
backup.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_backup_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
vaults = AccountRegionScopedDict()
vaults.set_scoped(
"123456789012",
"us-west-2",
"regional-vault",
{
"BackupVaultName": "regional-vault",
"BackupVaultArn": "arn:aws:backup:us-west-2:123456789012:backup-vault:regional-vault",
},
)
persistence.save_state("backup", {"vaults": vaults})
raw = _json.loads((tmp_path / "backup.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_vaults = persistence.load_state("backup")["vaults"]
assert loaded_vaults.get_scoped(
"123456789012", "us-west-2", "regional-vault"
)["BackupVaultName"] == "regional-vault"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("backup") is None
@pytest.mark.parametrize(
"legacy_account_scoped",
[True, False],
ids=["account-scoped", "plain-dict"],
)
def test_ecr_legacy_state_uses_parent_repository_region(
legacy_account_scoped,
):
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import ecr as mod
original_account = get_account_id()
original_region = get_region()
account_id = "000000000000"
boot_region = "us-east-1"
repository_region = "eu-west-1"
name = "legacy-repository"
orphan = "orphan-repository"
def legacy_store(values):
if not legacy_account_scoped:
return values
store = AccountScopedDict()
store._data.update(
{(account_id, key): value for key, value in values.items()}
)
return store
repositories = legacy_store(
{
name: {
"repositoryArn": (
f"arn:aws:ecr:{repository_region}:{account_id}:repository/{name}"
),
"repositoryName": name,
}
}
)
images = legacy_store(
{
name: [{"imageDigest": "sha256:image"}],
orphan: [{"imageDigest": "sha256:orphan"}],
}
)
lifecycle_policies = legacy_store({name: "legacy-lifecycle"})
repo_policies = legacy_store({name: "legacy-repository-policy"})
layer_blobs = legacy_store({name: {"sha256:layer": b"layer"}})
manifest_blobs = legacy_store(
{name: {"sha256:manifest": b"manifest"}}
)
mod.reset()
try:
set_request_account_id(account_id)
set_request_region(boot_region)
mod.load_persisted_state(
{
"repositories": repositories,
"images": images,
"lifecycle_policies": lifecycle_policies,
"repo_policies": repo_policies,
"layer_blobs": layer_blobs,
"manifest_blobs": manifest_blobs,
}
)
assert mod._repositories.get_scoped(
account_id, repository_region, name
)["repositoryName"] == name
assert mod._images.get_scoped(account_id, repository_region, name) == [
{"imageDigest": "sha256:image"}
]
assert mod._lifecycle_policies.get_scoped(
account_id, repository_region, name
) == "legacy-lifecycle"
assert mod._repo_policies.get_scoped(
account_id, repository_region, name
) == "legacy-repository-policy"
assert mod._layer_blobs.get_scoped(
account_id, repository_region, name
) == {"sha256:layer": b"layer"}
assert mod._manifest_blobs.get_scoped(
account_id, repository_region, name
) == {"sha256:manifest": b"manifest"}
assert mod._images.get_scoped(account_id, boot_region, orphan) == [
{"imageDigest": "sha256:orphan"}
]
finally:
mod.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_eks_round_trip():
def populate(mod):
mod._clusters["cluster-test"] = {"name": "cluster-test", "status": "ACTIVE"}
def observe(mod):
assert "cluster-test" in mod._clusters
_round_trip("eks", "eks", populate, observe)
def test_eks_access_policies_round_trip_outside_boot_region():
from ministack.core.responses import (
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
original_account = get_account_id()
original_region = get_region()
account_id = "000000000000"
boot_region = "us-east-1"
resource_region = "us-west-2"
cluster_name = "regional-cluster"
principal_arn = f"arn:aws:iam::{account_id}:role/regional-role"
entry_key = f"{cluster_name}\x00{principal_arn}"
policy_arn = (
"arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy"
)
policy_key = f"{entry_key}\x00{policy_arn}"
mod = _module("eks")
def populate(mod):
mod._access_entries.set_scoped(
account_id,
resource_region,
entry_key,
{
"clusterName": cluster_name,
"principalArn": principal_arn,
"accessEntryArn": (
f"arn:aws:eks:{resource_region}:{account_id}:"
f"access-entry/{cluster_name}/role/{account_id}/"
"regional-role/id"
),
},
)
mod._access_policies.set_scoped(
account_id,
resource_region,
policy_key,
{
"policyArn": policy_arn,
"accessScope": {"type": "cluster", "namespaces": []},
},
)
# Restore below runs in the boot scope, where this west-only store is
# false even though it contains persisted data in another region.
set_request_region(boot_region)
def observe(mod):
assert mod._access_policies.contains_scoped(
account_id, resource_region, policy_key
)
assert not mod._access_policies.contains_scoped(
account_id, boot_region, policy_key
)
try:
set_request_account_id(account_id)
set_request_region(resource_region)
_round_trip("eks", "eks", populate, observe)
finally:
mod.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_eks_legacy_state_uses_arn_and_parent_regions():
import json as _json
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import eks as mod
original_account = get_account_id()
original_region = get_region()
account_id = "000000000000"
boot_region = "us-east-1"
parent_region = "us-west-2"
child_arn_region = "eu-west-1"
cluster_name = "legacy-cluster"
principal_arn = f"arn:aws:iam::{account_id}:role/legacy-role"
entry_key = f"{cluster_name}\x00{principal_arn}"
policy_arn = (
"arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy"
)
policy_key = f"{entry_key}\x00{policy_arn}"
orphan_key = f"orphan-cluster\x00{principal_arn}\x00{policy_arn}"
legacy_clusters = AccountScopedDict()
legacy_clusters._data[(account_id, cluster_name)] = {
"name": cluster_name,
"arn": (
f"arn:aws:eks:{parent_region}:{account_id}:"
f"cluster/{cluster_name}"
),
"status": "ACTIVE",
"_port": 16443,
"_docker_id": "stale-container",
}
legacy_entries = AccountScopedDict()
legacy_entries._data[(account_id, entry_key)] = {
"clusterName": cluster_name,
"principalArn": principal_arn,
"accessEntryArn": (
f"arn:aws:eks:{child_arn_region}:{account_id}:"
f"access-entry/{cluster_name}/legacy-entry-id"
),
}
nodegroup_key = f"{cluster_name}/legacy-workers"
legacy_nodegroups = AccountScopedDict()
legacy_nodegroups._data[(account_id, nodegroup_key)] = {
"clusterName": cluster_name,
"nodegroupName": "legacy-workers",
"nodegroupArn": (
f"arn:aws:eks:{child_arn_region}:{account_id}:"
f"nodegroup/{cluster_name}/legacy-workers/id"
),
}
addon_key = f"{cluster_name}/vpc-cni"
legacy_addons = AccountScopedDict()
legacy_addons._data[(account_id, addon_key)] = {
"clusterName": cluster_name,
"addonName": "vpc-cni",
"addonArn": (
f"arn:aws:eks:{child_arn_region}:{account_id}:"
f"addon/{cluster_name}/vpc-cni/id"
),
}
legacy_policies = AccountScopedDict()
legacy_policies._data[(account_id, policy_key)] = {
"policyArn": policy_arn,
"accessScope": {"type": "cluster", "namespaces": []},
}
legacy_policies._data[(account_id, orphan_key)] = {
"policyArn": policy_arn,
"accessScope": {"type": "cluster", "namespaces": []},
}
idp_key = f"{cluster_name}\x00legacy-idp"
legacy_idp_configs = AccountScopedDict()
legacy_idp_configs._data[(account_id, idp_key)] = {
"clusterName": cluster_name,
"name": "legacy-idp",
"oidc": {
"clientId": "legacy-client",
"issuerUrl": "https://legacy.example.test",
},
"arn": (
f"arn:aws:eks:{child_arn_region}:{account_id}:"
f"identityproviderconfig/{cluster_name}/oidc/legacy-idp/id"
),
}
child_arns = (
legacy_nodegroups._data[(account_id, nodegroup_key)]["nodegroupArn"],
legacy_addons._data[(account_id, addon_key)]["addonArn"],
legacy_entries._data[(account_id, entry_key)]["accessEntryArn"],
legacy_idp_configs._data[(account_id, idp_key)]["arn"],
)
legacy_tags = AccountScopedDict()
for child_arn in child_arns:
legacy_tags._data[(account_id, child_arn)] = {"legacy": "true"}
mod.reset()
try:
set_request_account_id(account_id)
set_request_region(boot_region)
mod.load_persisted_state(
{
"clusters": legacy_clusters,
"nodegroups": legacy_nodegroups,
"addons": legacy_addons,
"access_entries": legacy_entries,
"access_policies": legacy_policies,
"idp_configs": legacy_idp_configs,
"tags": legacy_tags,
}
)
cluster = mod._clusters.get_scoped(
account_id, parent_region, cluster_name
)
assert cluster["_docker_id"] is None
assert cluster["endpoint"] == "https://localhost:16443"
assert mod._access_entries.contains_scoped(
account_id, parent_region, entry_key
)
assert mod._nodegroups.contains_scoped(
account_id, parent_region, nodegroup_key
)
assert mod._addons.contains_scoped(
account_id, parent_region, addon_key
)
assert mod._access_policies.contains_scoped(
account_id, parent_region, policy_key
)
assert not mod._access_policies.contains_scoped(
account_id, boot_region, policy_key
)
assert mod._access_policies.contains_scoped(
account_id, boot_region, orphan_key
)
assert mod._idp_configs.contains_scoped(
account_id, parent_region, idp_key
)
for store, key in (
(mod._nodegroups, nodegroup_key),
(mod._addons, addon_key),
(mod._access_entries, entry_key),
(mod._access_policies, policy_key),
(mod._idp_configs, idp_key),
):
assert not store.contains_scoped(account_id, child_arn_region, key)
# The legacy child records remain reachable through their normal API
# paths in the restored parent region even though their persisted ARNs
# were minted by cross-region legacy requests.
set_request_region(parent_region)
status, _, body = mod._list_nodegroups(cluster_name, {})
assert status == 200
assert _json.loads(body)["nodegroups"] == ["legacy-workers"]
status, _, body = mod._describe_nodegroup(
cluster_name, "legacy-workers"
)
assert status == 200
assert _json.loads(body)["nodegroup"]["nodegroupName"] == "legacy-workers"
status, _, body = mod._list_addons(cluster_name, {})
assert status == 200
assert _json.loads(body)["addons"] == ["vpc-cni"]
status, _, body = mod._describe_addon(cluster_name, "vpc-cni")
assert status == 200
assert _json.loads(body)["addon"]["addonName"] == "vpc-cni"
status, _, body = mod._list_access_entries(cluster_name, {})
assert status == 200
assert _json.loads(body)["accessEntries"] == [principal_arn]
status, _, body = mod._list_associated_access_policies(
cluster_name, principal_arn, {}
)
assert status == 200
assert _json.loads(body)["associatedAccessPolicies"][0][
"policyArn"
] == policy_arn
status, _, body = mod._describe_identity_provider_config(
cluster_name,
{
"identityProviderConfig": {
"type": "oidc",
"name": "legacy-idp",
}
},
)
assert status == 200
assert _json.loads(body)["identityProviderConfig"]["oidc"][
"identityProviderConfigName"
] == "legacy-idp"
# Tag APIs accept the original child ARN in the region where migration
# co-located the child with its parent. Keep that persisted ARN as the
# tag-store key so legacy tags survive the round trip.
for child_arn in child_arns:
status, _, body = mod._list_tags(child_arn)
assert status == 200
assert _json.loads(body)["tags"] == {"legacy": "true"}
nodegroup_arn = child_arns[0]
status, _, _body = mod._tag_resource(
nodegroup_arn, {"tags": {"owner": "platform"}}
)
assert status == 200
status, _, _body = mod._untag_resource(
nodegroup_arn, {"tagKeys": ["legacy"]}
)
assert status == 200
status, _, body = mod._list_tags(nodegroup_arn)
assert status == 200
assert _json.loads(body)["tags"] == {"owner": "platform"}
assert mod._tags.get(nodegroup_arn) == {"owner": "platform"}
finally:
mod.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_scheduler_round_trip():
# Production code keys _schedules by `f"{group}/{name}"` strings (see
# scheduler.py CreateSchedule etc.), not tuples — even though the
# pre-existing inline comment on the dict mis-describes the shape. Use
# the real production key shape so this test catches a regression that
# broke string-key serialisation.
from ministack.core.responses import get_region, set_request_region
original_region = get_region()
def populate(mod):
set_request_region("us-east-1")
mod._schedule_groups["default"] = {
"Arn": "arn:aws:scheduler:us-east-1:000000000000:schedule-group/default",
"Name": "default",
}
mod._schedules["default/sched-test"] = {
"Arn": "arn:aws:scheduler:us-east-1:000000000000:schedule/default/sched-test",
"Name": "sched-test",
"ScheduleExpression": "rate(1 hour)",
}
set_request_region("us-west-2")
mod._schedule_groups["default"] = {
"Arn": "arn:aws:scheduler:us-west-2:000000000000:schedule-group/default",
"Name": "default",
}
mod._schedules["default/sched-test"] = {
"Arn": "arn:aws:scheduler:us-west-2:000000000000:schedule/default/sched-test",
"Name": "sched-test",
"ScheduleExpression": "rate(2 hours)",
}
def observe(mod):
set_request_region("us-east-1")
assert mod._schedules["default/sched-test"]["ScheduleExpression"] == "rate(1 hour)"
set_request_region("us-west-2")
assert mod._schedules["default/sched-test"]["ScheduleExpression"] == "rate(2 hours)"
try:
_round_trip("scheduler", "scheduler", populate, observe)
finally:
set_request_region(original_region)
def test_scheduler_legacy_account_scoped_state_uses_resource_arn_region():
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import scheduler as mod
original_account = get_account_id()
original_region = get_region()
account_id = "000000000000"
region = "us-west-2"
group = "legacy-group"
schedule_key = f"{group}/legacy-schedule"
legacy_groups = AccountScopedDict()
legacy_groups._data[(account_id, group)] = {
"Arn": f"arn:aws:scheduler:{region}:{account_id}:schedule-group/{group}",
"Name": group,
}
legacy_schedules = AccountScopedDict()
legacy_schedules._data[(account_id, schedule_key)] = {
"Arn": f"arn:aws:scheduler:{region}:{account_id}:schedule/{schedule_key}",
"Name": "legacy-schedule",
"GroupName": group,
}
mod.reset()
try:
set_request_account_id(account_id)
set_request_region("us-east-1")
mod.load_persisted_state(
{"schedule_groups": legacy_groups, "schedules": legacy_schedules}
)
assert mod._schedule_groups.get_scoped(account_id, "us-east-1", group) is None
assert mod._schedules.get_scoped(account_id, "us-east-1", schedule_key) is None
assert mod._schedule_groups.get_scoped(account_id, region, group)["Name"] == group
assert mod._schedules.get_scoped(account_id, region, schedule_key)["Name"] == (
"legacy-schedule"
)
finally:
mod.reset()
set_request_account_id(original_account)
set_request_region(original_region)
@pytest.mark.parametrize(
"legacy_account_scoped",
[True, False],
ids=["legacy-account-scope", "current-region-scope"],
)
def test_mwaa_restore_preserves_resource_region_outside_boot_scope(
monkeypatch,
legacy_account_scoped,
):
from types import SimpleNamespace
from ministack.core.responses import (
AccountRegionScopedDict,
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import mwaa as mod
class ImmediateThread:
def __init__(self, target, args, daemon):
self.target = target
self.args = args
self.daemon = daemon
def start(self):
self.target(*self.args)
original_account = get_account_id()
original_region = get_region()
account_id = "111111111111"
boot_region = "us-east-1"
resource_region = "us-west-2"
env_name = "warm-boot-environment"
env = {
"Name": env_name,
"Arn": f"arn:aws:airflow:{resource_region}:{account_id}:environment/{env_name}",
"Status": "AVAILABLE",
"_docker_container_id": "stale-container",
}
restored = (
AccountScopedDict()
if legacy_account_scoped
else AccountRegionScopedDict()
)
if legacy_account_scoped:
restored._data[(account_id, env_name)] = env
persistence.save_state("mwaa", {"environments": restored})
loaded = persistence.load_state("mwaa")
assert isinstance(loaded["environments"], AccountScopedDict)
restored = loaded["environments"]
else:
restored._data[(account_id, resource_region, env_name)] = env
restart_args = []
monkeypatch.setattr(mod, "_get_docker", lambda: None)
monkeypatch.setattr(mod, "_start_airflow_container", lambda *args: restart_args.append(args))
monkeypatch.setattr(mod, "threading", SimpleNamespace(Thread=ImmediateThread))
mod.reset()
try:
set_request_account_id(account_id)
set_request_region(boot_region)
mod.load_persisted_state({"environments": restored})
assert mod._environments.get_scoped(account_id, boot_region, env_name) is None
restored_env = mod._environments.get_scoped(
account_id,
resource_region,
env_name,
)
assert restored_env["Status"] == "CREATING"
assert restored_env["_docker_container_id"] is None
expected_volume_prefix = (
f"ministack-mwaa-{env_name}"
if legacy_account_scoped
else f"ministack-mwaa-{resource_region}-{env_name}"
)
if legacy_account_scoped:
assert restored_env["_docker_dags_volume_name"] == (
f"{expected_volume_prefix}-dags"
)
assert restored_env["_docker_db_volume_name"] == (
f"{expected_volume_prefix}-db"
)
else:
assert "_docker_dags_volume_name" not in restored_env
assert "_docker_db_volume_name" not in restored_env
assert restart_args == [
(account_id, resource_region, env_name, restored_env)
]
finally:
mod.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_pipes_round_trip():
# Use a complete pipe record matching `register_pipe()` shape so the
# background poller (which the restore path may start) doesn't blow up
# on KeyError if it iterates this entry. Source/Target are intentionally
# non-DDB/non-SNS so `_poll_once` skips them quickly.
pipe_arn = "arn:aws:pipes:us-east-1:000000000000:pipe/pipe-test"
def populate(mod):
mod._pipes["pipe-test"] = {
"Name": "pipe-test",
"Arn": pipe_arn,
"RoleArn": "",
"Source": "arn:aws:sqs:us-east-1:000000000000:irrelevant",
"Target": "arn:aws:sqs:us-east-1:000000000000:irrelevant",
"DesiredState": "STOPPED",
"CurrentState": "STOPPED",
"StartingPosition": "LATEST",
"Tags": {},
"CreationTime": 0,
}
mod._positions[pipe_arn] = 0
def observe(mod):
assert "pipe-test" in mod._pipes
assert mod._positions.get(pipe_arn) == 0
_round_trip("pipes", "pipes", populate, observe)
def test_pipes_restore_starts_poller_for_running_pipes(monkeypatch):
"""When `load_persisted_state` reloads RUNNING pipes, the background
poller must be (re)started so events keep flowing after warm-boot."""
mod = _module("pipes")
mod.reset()
# Reset the poller flag so this test is independent of execution order.
monkeypatch.setattr(mod, "_poller_started", False)
pipe_arn = "arn:aws:pipes:us-east-1:000000000000:pipe/poller-test"
result = mod.load_persisted_state({
"pipes": {
"poller-test": {
"Name": "poller-test",
"Arn": pipe_arn,
"RoleArn": "",
"Source": "arn:aws:sqs:us-east-1:000000000000:irrelevant",
"Target": "arn:aws:sqs:us-east-1:000000000000:irrelevant",
"DesiredState": "RUNNING",
"CurrentState": "RUNNING",
"StartingPosition": "LATEST",
"Tags": {},
"CreationTime": 0,
},
},
"positions": {pipe_arn: 0},
})
assert mod._poller_started, (
"load_persisted_state() did not start the pipes poller for a RUNNING pipe — "
"warm-booted pipes would silently stop forwarding events."
)
assert result is None
mod.reset()
def test_lambda_load_starts_event_source_mapping_poller(monkeypatch):
"""Warm boot resumes the Lambda poller only through the loader API."""
mod = _module("lambda_svc")
mod.reset()
calls = []
monkeypatch.setattr(mod, "_ensure_poller", lambda: calls.append(None))
mod.load_persisted_state({
"esms": {
"warm-boot-esm": {
"UUID": "warm-boot-esm",
"FunctionArn": (
"arn:aws:lambda:us-east-1:000000000000:function:warm-boot"
),
},
},
})
assert calls == [None]
assert mod._esms.has_any()
mod.reset()
def test_registry_loader_dispatches_saved_state_to_the_declared_module(monkeypatch):
"""The central loader imports only saved modules and calls their contract."""
import ministack.app as app
class Module:
def __init__(self):
self.loaded = []
def load_persisted_state(self, data):
self.loaded.append(data)
module = Module()
monkeypatch.setattr(app, "_state_map", {"saved": "module", "empty": "other"})
monkeypatch.setattr(app, "load_state", lambda key: {"value": key} if key == "saved" else None)
monkeypatch.setattr(app, "_get_module", lambda name: module if name == "module" else pytest.fail(name))
app._load_persisted_state()
assert module.loaded == [{"value": "saved"}]
@pytest.mark.parametrize("legacy", [False, True])
@pytest.mark.parametrize("objects_first", [False, True])
def test_s3_central_restore_preserves_metadata_and_objects(monkeypatch, tmp_path, legacy, objects_first):
"""Object loading and central metadata restoration work in either order."""
import ministack.app as app
from ministack.core.responses import AccountScopedDict, request_scope
from ministack.services import s3
monkeypatch.setattr(s3, "_buckets", AccountScopedDict())
monkeypatch.setattr(s3, "DATA_DIR", str(tmp_path / "objects"))
monkeypatch.setattr(s3, "S3_PERSIST", True)
monkeypatch.setattr(app, "_state_map", {"s3": "s3"})
monkeypatch.setattr(app, "_loaded_modules", {})
accounts = ["111111111111"] if legacy else ["111111111111", "222222222222"]
metadata = {} if legacy else AccountScopedDict()
for account in accounts:
bucket_meta = {
"created": "2020-01-01T00:00:00Z",
"region": "eu-west-1",
"_ownership_controls": "<OwnershipControls><Rule><ObjectOwnership>BucketOwnerEnforced</ObjectOwnership></Rule></OwnershipControls>",
"_public_access_block": "<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls></PublicAccessBlockConfiguration>",
}
for name in ("bucket", "empty-bucket"):
if legacy:
metadata[name] = dict(bucket_meta)
else:
metadata._data[(account, name)] = dict(bucket_meta)
object_path = tmp_path / "objects" / account / "bucket" / "key"
object_path.parent.mkdir(parents=True)
object_path.write_bytes(account.encode())
persistence.save_state("s3", {"buckets_meta": metadata})
with request_scope(accounts[0], "us-east-1"):
if objects_first:
s3._load_persisted_data()
app._load_persisted_state()
if not objects_first:
s3._load_persisted_data()
for account in accounts:
with request_scope(account, "us-east-1"):
for name in ("bucket", "empty-bucket"):
bucket = s3._buckets[name]
assert {key: value for key, value in bucket.items() if key != "objects"} == bucket_meta
obj = s3._buckets["bucket"]["objects"]["key"]
assert s3._read_body("bucket", "key", obj) == account.encode()
assert s3._buckets["empty-bucket"]["objects"] == {}
# ── PERSIST_STATE gating ──────────────────────────────────────────────
@pytest.mark.parametrize("svc_key", [
"autoscaling", "backup", "eks", "scheduler", "pipes",
])
def test_load_state_is_noop_when_persist_state_disabled(monkeypatch, svc_key, tmp_path):
"""When PERSIST_STATE=0, load_state() must return None without touching
disk and without invoking load_persisted_state(). Catches a regression
where a service module accidentally loads state unconditionally."""
monkeypatch.setattr(persistence, "PERSIST_STATE", False)
# Pre-write a state file that *would* succeed if persistence were on,
# so we can assert that it is NOT consumed.
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
bogus_path = tmp_path / f"{svc_key}.json"
bogus_path.write_text('{"would_have_been_restored": true}')
result = persistence.load_state(svc_key)
assert result is None, (
f"load_state({svc_key!r}) returned non-None even though "
"PERSIST_STATE is False — restore must be gated."
)
# ========== from test_state_dict_persistence.py ==========
# Companion to the symmetry tests above. Symmetry tests check that
# *every service* participates in get_state/restore_state. These tests
# check that within services, *every AccountScopedDict mutated by the
# public API* is captured — the 'dict dropped from get_state' bug pattern.
import importlib
import pytest
from ministack.core import persistence
def _get_module(mod_name):
return importlib.import_module(f"ministack.services.{mod_name}")
@pytest.fixture(autouse=True)
def _enable_persistence_dict(monkeypatch, tmp_path):
"""Force PERSIST_STATE on and point STATE_DIR at a tmp dir for the
duration of each test so save_state / load_state actually write and
read JSON files instead of short-circuiting."""
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
def _round_trip_dict(mod, svc_key):
"""Simulate a full warm-boot through the on-disk JSON path.
Going through `save_state` / `load_state` (rather than calling
`get_state` / `restore_state` directly in-memory) catches encoder
/ decoder regressions AND import-order bugs (a `restore_state`
that references a globals-only symbol declared further down the
module would NameError on real warm-boot but pass an in-memory
test that already has the symbol bound)."""
persistence.save_state(svc_key, mod.get_state())
mod.reset()
loaded = persistence.load_state(svc_key)
assert loaded is not None, (
f"persistence.load_state({svc_key!r}) returned None — state "
"file was not written by save_state()."
)
mod.load_persisted_state(loaded)
# ── secretsmanager._resource_policies ──────────────────────────────────
def test_secretsmanager_resource_policies_survive_warm_boot():
"""`PutResourcePolicy` writes to `_resource_policies`, but if that
dict is missing from `get_state()` the policy is gone after restart.
Terraform `aws_secretsmanager_secret_policy` would silently drop."""
mod = _get_module("secretsmanager")
mod.reset()
arn = "arn:aws:secretsmanager:us-east-1:000000000000:secret:my-secret-AbCdEf"
mod._resource_policies[arn] = '{"Version":"2012-10-17","Statement":[]}'
_round_trip_dict(mod, "secretsmanager")
assert mod._resource_policies.get(arn) == '{"Version":"2012-10-17","Statement":[]}', (
"Resource policy lost across get_state → restore_state — "
"_resource_policies must be in both."
)
mod.reset()
# ── kinesis._consumers ─────────────────────────────────────────────────
def test_kinesis_consumers_survive_warm_boot():
"""`RegisterStreamConsumer` writes to `_consumers`. Without
persistence symmetry, every enhanced fan-out registration is lost on
restart and `DescribeStreamConsumer` returns ResourceNotFoundException."""
mod = _get_module("kinesis")
mod.reset()
consumer_arn = (
"arn:aws:kinesis:us-east-1:000000000000:stream/my-stream/consumer/c1:123"
)
mod._consumers[consumer_arn] = {
"ConsumerARN": consumer_arn,
"ConsumerName": "c1",
"ConsumerStatus": "ACTIVE",
"StreamARN": "arn:aws:kinesis:us-east-1:000000000000:stream/my-stream",
"ConsumerCreationTimestamp": 1700000000.0,
}
_round_trip_dict(mod, "kinesis")
assert consumer_arn in mod._consumers, (
"Kinesis consumer lost across get_state → restore_state — "
"_consumers must be in both."
)
mod.reset()
def test_kinesis_shard_iterators_survive_warm_boot_in_original_scope():
"""Live shard iterators must round-trip through JSON persistence without
becoming visible to other accounts or regions."""
import json
import time
from ministack.core.responses import (
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
mod = _get_module("kinesis")
original_account = get_account_id()
original_region = get_region()
account_id = "111111111111"
region = "us-west-2"
stream_name = "warm-boot-stream"
stream_arn = f"arn:aws:kinesis:{region}:{account_id}:stream/{stream_name}"
shard_id = "shardId-000000000000"
token = "live-iterator-token"
now = time.time()
mod.reset()
try:
set_request_account_id(account_id)
set_request_region(region)
mod._streams[stream_name] = {
"StreamName": stream_name,
"StreamARN": stream_arn,
"StreamStatus": "ACTIVE",
"StreamModeDetails": {"StreamMode": "PROVISIONED"},
"RetentionPeriodHours": 24,
"shards": {
shard_id: {
"records": [{
"SequenceNumber": "1",
"ApproximateArrivalTimestamp": now,
"Data": b"warm-boot-record",
"PartitionKey": "pk1",
}],
"starting_hash_key": "0",
"ending_hash_key": str(2**128 - 1),
"starting_sequence_number": "1",
"parent_shard_id": None,
"adjacent_parent_shard_id": None,
},
},
"tags": {},
"CreationTimestamp": now,
"EncryptionType": "NONE",
}
mod._shard_iterators[token] = {
"stream": stream_name,
"stream_arn": stream_arn,
"shard_id": shard_id,
"position": 0,
"created_at": now,
}
_round_trip_dict(mod, "kinesis")
assert token in mod._shard_iterators
status, _, body = mod._get_records({"ShardIterator": token, "Limit": 1})
assert status == 200
payload = json.loads(body)
assert payload["Records"][0]["PartitionKey"] == "pk1"
assert payload["NextShardIterator"] in mod._shard_iterators
set_request_region("us-east-1")
assert token not in mod._shard_iterators
status, _, body = mod._get_records({"ShardIterator": token})
assert status == 400
assert json.loads(body)["__type"] == "ExpiredIteratorException"
set_request_account_id("222222222222")
set_request_region(region)
assert token not in mod._shard_iterators
finally:
mod.reset()
set_request_account_id(original_account)
set_request_region(original_region)
# ── ecs._attributes ────────────────────────────────────────────────────
def test_ecs_attributes_survive_warm_boot():
"""`PutAttributes` writes to `_attributes`. Lost on restart without
persistence wiring."""
mod = _get_module("ecs")
mod.reset()
mod._attributes["i-deadbeef:my-attr"] = {
"name": "my-attr",
"value": "v1",
"targetType": "container-instance",
"targetId": "i-deadbeef",
}
_round_trip_dict(mod, "ecs")
assert "i-deadbeef:my-attr" in mod._attributes, (
"ECS attribute lost across get_state → restore_state — "
"_attributes must be in both."
)
mod.reset()
def test_ecs_region_scoped_state_survives_warm_boot(monkeypatch):
"""ECS v2 persistence retains every resource's account and region scope."""
from ministack.core.responses import (
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
mod = _get_module("ecs")
original_account = get_account_id()
original_region = get_region()
account_id = "111111111111"
cluster_name = "warm-boot-cluster"
family = "warm-boot-family"
monkeypatch.setattr(mod, "_get_docker", lambda: None)
mod.reset()
try:
set_request_account_id(account_id)
for region, revision in (("us-east-1", 2), ("us-west-2", 1)):
set_request_region(region)
cluster_arn = f"arn:aws:ecs:{region}:{account_id}:cluster/{cluster_name}"
td_key = f"{family}:{revision}"
td_arn = f"arn:aws:ecs:{region}:{account_id}:task-definition/{td_key}"
service_key = f"{cluster_name}/warm-boot-service"
service_arn = (
f"arn:aws:ecs:{region}:{account_id}:service/"
f"{cluster_name}/warm-boot-service"
)
task_arn = f"arn:aws:ecs:{region}:{account_id}:task/{cluster_name}/{region}"
cp_arn = f"arn:aws:ecs:{region}:{account_id}:capacity-provider/warm-boot-cp"
mod._clusters[cluster_name] = {
"clusterArn": cluster_arn,
"clusterName": cluster_name,
"status": "ACTIVE",
}
mod._task_defs[td_key] = {
"taskDefinitionArn": td_arn,
"family": family,
"revision": revision,
}
mod._task_def_latest[family] = revision
mod._services[service_key] = {
"serviceArn": service_arn,
"serviceName": "warm-boot-service",
"clusterArn": cluster_arn,
}
mod._tasks[task_arn] = {
"taskArn": task_arn,
"clusterArn": cluster_arn,
"lastStatus": "RUNNING",
"_docker_ids": [f"stale-{region}"],
}
mod._capacity_providers["warm-boot-cp"] = {
"capacityProviderArn": cp_arn,
"name": "warm-boot-cp",
}
mod._attributes["i-warm:zone"] = {
"targetId": "i-warm",
"name": "zone",
"value": region,
}
mod._account_settings["containerInsights"] = region
mod._tags[cluster_arn] = [{"key": "region", "value": region}]
_round_trip_dict(mod, "ecs")
for region, revision in (("us-east-1", 2), ("us-west-2", 1)):
cluster_arn = f"arn:aws:ecs:{region}:{account_id}:cluster/{cluster_name}"
task_arn = f"arn:aws:ecs:{region}:{account_id}:task/{cluster_name}/{region}"
assert mod._clusters.get_scoped(account_id, region, cluster_name)["clusterArn"] == (
cluster_arn
)
assert mod._task_def_latest.get_scoped(account_id, region, family) == revision
assert mod._account_settings.get_scoped(
account_id, region, "containerInsights"
) == region
assert mod._attributes.get_scoped(account_id, region, "i-warm:zone")["value"] == (
region
)
restored_task = mod._tasks.get_scoped(account_id, region, task_arn)
assert restored_task["lastStatus"] == "STOPPED"
assert restored_task["_docker_ids"] == []
assert mod._tags.get_scoped(account_id, region, cluster_arn) == [
{"key": "region", "value": region}
]
finally:
mod.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_ecs_legacy_account_scoped_state_migrates_by_arn(monkeypatch, tmp_path):
"""Legacy ECS state adopts ARN regions and boot-scopes ARN-less stores."""
import json as _json
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
mod = _get_module("ecs")
original_account = get_account_id()
original_region = get_region()
account_id = "111111111111"
boot_region = "us-east-1"
resource_region = "us-west-2"
cluster_name = "legacy-cluster"
family = "legacy-family"
cluster_arn = f"arn:aws:ecs:{resource_region}:{account_id}:cluster/{cluster_name}"
service_key = f"{cluster_name}/legacy-service"
service_arn = (
f"arn:aws:ecs:{resource_region}:{account_id}:service/"
f"{cluster_name}/legacy-service"
)
task_arn = (
f"arn:aws:ecs:{resource_region}:{account_id}:task/"
f"{cluster_name}/legacy-task"
)
def scoped(key, value):
store = AccountScopedDict()
store._data[(account_id, key)] = value
return store
legacy_payload = {
"clusters": scoped(cluster_name, {
"clusterArn": cluster_arn,
"clusterName": cluster_name,
"status": "ACTIVE",
}),
"task_defs": scoped(family + ":3", {
"taskDefinitionArn": (
f"arn:aws:ecs:{resource_region}:{account_id}:"
f"task-definition/{family}:3"
),
"family": family,
"revision": 3,
}),
"task_def_latest": scoped(family, 3),
"services": scoped(service_key, {
"serviceArn": service_arn,
"serviceName": "legacy-service",
"clusterArn": cluster_arn,
}),
"tasks": scoped(task_arn, {
"taskArn": task_arn,
"clusterArn": cluster_arn,
"lastStatus": "RUNNING",
"_docker_ids": ["stale-container"],
}),
"account_settings": scoped("containerInsights", "enabled"),
"attributes": scoped("i-legacy:zone", {
"targetId": "i-legacy",
"name": "zone",
"value": "legacy",
}),
}
monkeypatch.setattr(mod, "_get_docker", lambda: None)
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
mod.reset()
try:
set_request_account_id(account_id)
set_request_region(boot_region)
with open(tmp_path / "ecs.json", "w") as f:
_json.dump(legacy_payload, f, default=persistence._json_default)
loaded = persistence.load_state("ecs")
assert isinstance(loaded["tasks"], AccountScopedDict)
mod.load_persisted_state(loaded)
assert mod._clusters.get_scoped(account_id, resource_region, cluster_name)[
"clusterArn"
] == cluster_arn
assert mod._services.get_scoped(account_id, resource_region, service_key)[
"serviceArn"
] == service_arn
restored_task = mod._tasks.get_scoped(account_id, resource_region, task_arn)
assert restored_task["lastStatus"] == "STOPPED"
assert restored_task["_docker_ids"] == []
assert mod._task_def_latest.get_scoped(account_id, boot_region, family) is None
assert mod._task_def_latest.get_scoped(account_id, resource_region, family) == 3
set_request_region(resource_region)
described = _json.loads(
mod._describe_task_definition({"taskDefinition": family})[2]
)["taskDefinition"]
assert described["taskDefinitionArn"].endswith(
f"task-definition/{family}:3"
)
registered = _json.loads(mod._register_task_definition({
"family": family,
"containerDefinitions": [{"name": "app", "image": "busybox"}],
})[2])["taskDefinition"]
assert registered["revision"] == 4
assert mod._task_defs.get_scoped(
account_id, resource_region, family + ":3"
)["taskDefinitionArn"] == described["taskDefinitionArn"]
assert mod._task_defs.get_scoped(
account_id, resource_region, family + ":4"
)["taskDefinitionArn"] == registered["taskDefinitionArn"]
# Other ARN-less legacy stores intentionally adopt the boot region.
assert mod._account_settings.get_scoped(
account_id, boot_region, "containerInsights"
) == "enabled"
assert mod._attributes.get_scoped(account_id, boot_region, "i-legacy:zone")[
"value"
] == "legacy"
finally:
mod.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_ecs_legacy_task_revision_migration_reconstructs_each_region(monkeypatch):
"""Legacy shared counters do not reset revisions in other ARN regions."""
import json as _json
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
mod = _get_module("ecs")
original_account = get_account_id()
original_region = get_region()
account_id = "111111111111"
boot_region = "us-east-1"
other_region = "us-west-2"
family = "multi-region-legacy-family"
legacy_task_defs = AccountScopedDict()
for region, revision, marker in (
(boot_region, 1, "east-original"),
(other_region, 2, "west-original"),
):
key = f"{family}:{revision}"
legacy_task_defs._data[(account_id, key)] = {
"taskDefinitionArn": (
f"arn:aws:ecs:{region}:{account_id}:task-definition/{key}"
),
"family": family,
"revision": revision,
"status": "ACTIVE",
"marker": marker,
}
legacy_latest = AccountScopedDict()
legacy_latest._data[(account_id, family)] = 2
monkeypatch.setattr(mod, "_get_docker", lambda: None)
mod.reset()
try:
set_request_account_id(account_id)
set_request_region(boot_region)
mod.load_persisted_state({
"task_defs": legacy_task_defs,
"task_def_latest": legacy_latest,
})
assert mod._task_def_latest.get_scoped(
account_id, boot_region, family
) == 1
assert mod._task_def_latest.get_scoped(
account_id, other_region, family
) == 2
for region, restored_revision, next_revision in (
(boot_region, 1, 2),
(other_region, 2, 3),
):
set_request_region(region)
status, _, body = mod._describe_task_definition({
"taskDefinition": family,
})
assert status == 200
described = _json.loads(body)["taskDefinition"]
assert described["revision"] == restored_revision
assert described["taskDefinitionArn"].startswith(
f"arn:aws:ecs:{region}:{account_id}:"
)
status, _, body = mod._register_task_definition({
"family": family,
"containerDefinitions": [{"name": "app", "image": "busybox"}],
})
assert status == 200
assert _json.loads(body)["taskDefinition"]["revision"] == next_revision
assert mod._task_defs.get_scoped(
account_id, boot_region, f"{family}:1"
)["marker"] == "east-original"
assert mod._task_defs.get_scoped(
account_id, other_region, f"{family}:2"
)["marker"] == "west-original"
finally:
mod.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_ecs_plain_dict_tasks_migrate_to_arn_region(monkeypatch):
"""Oldest bare-dict task state restores under the task ARN region."""
from ministack.core.responses import (
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
mod = _get_module("ecs")
original_account = get_account_id()
original_region = get_region()
account_id = "111111111111"
boot_region = "us-east-1"
resource_region = "us-west-2"
task_arn = f"arn:aws:ecs:{resource_region}:{account_id}:task/legacy-cluster/legacy-task"
monkeypatch.setattr(mod, "_get_docker", lambda: None)
mod.reset()
try:
set_request_account_id(account_id)
set_request_region(boot_region)
mod.load_persisted_state(
{
"tasks": {
task_arn: {
"taskArn": task_arn,
"lastStatus": "RUNNING",
"_docker_ids": ["stale-container"],
},
},
}
)
restored_task = mod._tasks.get_scoped(account_id, resource_region, task_arn)
assert restored_task["lastStatus"] == "STOPPED"
assert restored_task["_docker_ids"] == []
assert mod._tasks.get_scoped(account_id, boot_region, task_arn) is None
finally:
mod.reset()
set_request_account_id(original_account)
set_request_region(original_region)
# ── sns._platform_applications + sns._platform_endpoints ──────────────
def test_sns_platform_applications_survive_warm_boot():
"""`CreatePlatformApplication` writes to `_platform_applications`.
Mobile push topology is lost on restart without persistence wiring."""
mod = _get_module("sns")
mod.reset()
app_arn = "arn:aws:sns:us-east-1:000000000000:app/GCM/MyApp"
mod._platform_applications[app_arn] = {
"PlatformApplicationArn": app_arn,
"Attributes": {"Platform": "GCM"},
}
_round_trip_dict(mod, "sns")
assert app_arn in mod._platform_applications, (
"SNS platform application lost across get_state → restore_state — "
"_platform_applications must be in both."
)
mod.reset()
def test_sns_platform_endpoints_survive_warm_boot():
"""`CreatePlatformEndpoint` writes to `_platform_endpoints`."""
mod = _get_module("sns")
mod.reset()
ep_arn = "arn:aws:sns:us-east-1:000000000000:endpoint/GCM/MyApp/abc"
mod._platform_endpoints[ep_arn] = {
"EndpointArn": ep_arn,
"Token": "device-token-xyz",
"Enabled": "true",
}
_round_trip_dict(mod, "sns")
assert ep_arn in mod._platform_endpoints, (
"SNS platform endpoint lost across get_state → restore_state — "
"_platform_endpoints must be in both."
)
mod.reset()
def test_sns_region_scoped_stores_survive_warm_boot_in_original_scope():
"""SNS topic, subscription, application, and endpoint stores stay scoped
after the real JSON persistence path."""
from ministack.core.responses import (
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
mod = _get_module("sns")
mod.reset()
original_account = get_account_id()
original_region = get_region()
try:
set_request_account_id("111111111111")
set_request_region("us-west-2")
topic_arn = "arn:aws:sns:us-west-2:111111111111:persisted-topic"
sub_arn = f"{topic_arn}:sub-1"
app_arn = "arn:aws:sns:us-west-2:111111111111:app/GCM/PersistedApp"
endpoint_arn = f"{app_arn}/endpoint-1"
subscription = {
"arn": sub_arn,
"protocol": "email",
"endpoint": "[email protected]",
"confirmed": True,
"topic_arn": topic_arn,
"owner": "111111111111",
"attributes": {"SubscriptionArn": sub_arn, "TopicArn": topic_arn},
}
mod._topics[topic_arn] = {
"name": "persisted-topic",
"arn": topic_arn,
"attributes": {"TopicArn": topic_arn, "Owner": "111111111111"},
"subscriptions": [subscription],
"messages": [],
"tags": {},
}
mod._sub_arn_to_topic[sub_arn] = topic_arn
mod._platform_applications[app_arn] = {
"arn": app_arn,
"name": "PersistedApp",
"platform": "GCM",
"attributes": {},
}
mod._platform_endpoints[endpoint_arn] = {
"arn": endpoint_arn,
"application_arn": app_arn,
"attributes": {"Token": "persisted-token", "Enabled": "true"},
}
_round_trip_dict(mod, "sns")
assert mod._topics[topic_arn]["subscriptions"][0]["arn"] == sub_arn
assert mod._sub_arn_to_topic[sub_arn] == topic_arn
assert mod._platform_applications[app_arn]["arn"] == app_arn
assert mod._platform_endpoints[endpoint_arn]["attributes"]["Token"] == "persisted-token"
set_request_region("us-east-1")
assert mod._topics.get(topic_arn) is None
assert mod._sub_arn_to_topic.get(sub_arn) is None
assert mod._platform_applications.get(app_arn) is None
assert mod._platform_endpoints.get(endpoint_arn) is None
set_request_account_id("222222222222")
set_request_region("us-west-2")
assert mod._topics.get(topic_arn) is None
assert mod._sub_arn_to_topic.get(sub_arn) is None
assert mod._platform_applications.get(app_arn) is None
assert mod._platform_endpoints.get(endpoint_arn) is None
finally:
mod.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_eventbridge_region_scoped_stores_survive_warm_boot_in_original_scope():
"""EventBridge regional stores remain in their original account/region
after the real JSON persistence path."""
from ministack.core.responses import (
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
mod = _get_module("eventbridge")
mod.reset()
original_account = get_account_id()
original_region = get_region()
try:
set_request_account_id("111111111111")
set_request_region("us-west-2")
now = mod._now_ts()
bus_name = "persisted-bus"
rule_name = "persisted-rule"
archive_name = "persisted-archive"
replay_name = "persisted-replay"
endpoint_name = "persisted-endpoint"
connection_name = "persisted-connection"
api_destination_name = "persisted-api"
partner_name = "persisted-partner"
bus_arn = f"arn:aws:events:us-west-2:111111111111:event-bus/{bus_name}"
rule_key = mod._rule_key(rule_name, bus_name)
rule_arn = f"arn:aws:events:us-west-2:111111111111:rule/{bus_name}/{rule_name}"
archive_arn = f"arn:aws:events:us-west-2:111111111111:archive/{archive_name}"
replay_arn = f"arn:aws:events:us-west-2:111111111111:replay/{replay_name}"
endpoint_arn = f"arn:aws:events:us-west-2:111111111111:endpoint/{endpoint_name}"
connection_arn = f"arn:aws:events:us-west-2:111111111111:connection/{connection_name}"
api_destination_arn = (
f"arn:aws:events:us-west-2:111111111111:api-destination/{api_destination_name}"
)
partner_arn = f"arn:aws:events:us-west-2:222222222222:event-source/{partner_name}"
mod._event_buses[bus_name] = {
"Name": bus_name,
"Arn": bus_arn,
"CreationTime": now,
"LastModifiedTime": now,
}
mod._rules[rule_key] = {
"Name": rule_name,
"Arn": rule_arn,
"EventBusName": bus_name,
"ScheduleExpression": "rate(5 minutes)",
"State": "ENABLED",
"CreationTime": now,
}
mod._targets[rule_key] = [
{
"Id": "persisted-target",
"Arn": "arn:aws:sqs:us-west-2:111111111111:persisted-queue",
}
]
mod._tags[rule_arn] = {"env": "test"}
mod._archives[archive_name] = {
"ArchiveName": archive_name,
"ArchiveArn": archive_arn,
"EventSourceArn": bus_arn,
"State": "ENABLED",
"CreationTime": now,
"EventCount": 0,
"Events": [],
}
mod._replays[replay_name] = {
"ReplayName": replay_name,
"ReplayArn": replay_arn,
"EventSourceArn": archive_arn,
"Destination": {"Arn": bus_arn},
"State": "COMPLETED",
"ReplayStartTime": now,
"ReplayEndTime": now,
}
mod._endpoints[endpoint_name] = {
"Name": endpoint_name,
"Arn": endpoint_arn,
"EndpointUrl": f"https://{endpoint_name}.global-events.us-west-2.amazonaws.com",
"State": "ACTIVE",
"CreationTime": now,
"LastModifiedTime": now,
}
mod._event_bus_policies[bus_name] = {
"Version": "2012-10-17",
"Statement": [{"Sid": "Allow", "Resource": bus_arn}],
}
mod._connections[connection_name] = {
"Name": connection_name,
"ConnectionArn": connection_arn,
"ConnectionState": "AUTHORIZED",
"AuthorizationType": "API_KEY",
"CreationTime": now,
"LastModifiedTime": now,
}
mod._api_destinations[api_destination_name] = {
"Name": api_destination_name,
"ApiDestinationArn": api_destination_arn,
"ApiDestinationState": "ACTIVE",
"ConnectionArn": connection_arn,
"InvocationEndpoint": "https://example.com",
"HttpMethod": "POST",
"CreationTime": now,
"LastModifiedTime": now,
}
mod._partner_event_sources[mod._partner_key("222222222222", partner_name)] = {
"Name": partner_name,
"Account": "222222222222",
"EventSourceArn": partner_arn,
}
_round_trip_dict(mod, "eventbridge")
assert mod._event_buses[bus_name]["Arn"] == bus_arn
assert mod._rules[rule_key]["Arn"] == rule_arn
assert mod._targets[rule_key][0]["Id"] == "persisted-target"
assert mod._tags[rule_arn]["env"] == "test"
assert mod._archives[archive_name]["ArchiveArn"] == archive_arn
assert mod._replays[replay_name]["ReplayArn"] == replay_arn
assert mod._endpoints[endpoint_name]["Arn"] == endpoint_arn
assert mod._event_bus_policies[bus_name]["Statement"][0]["Resource"] == bus_arn
assert mod._connections[connection_name]["ConnectionArn"] == connection_arn
assert mod._api_destinations[api_destination_name]["ApiDestinationArn"] == api_destination_arn
partner_key = mod._partner_key("222222222222", partner_name)
assert mod._partner_event_sources[partner_key]["EventSourceArn"] == partner_arn
set_request_region("us-east-1")
assert mod._event_buses.get(bus_name) is None
assert mod._rules.get(rule_key) is None
assert mod._targets.get(rule_key) is None
assert mod._tags.get(rule_arn) is None
assert mod._archives.get(archive_name) is None
assert mod._replays.get(replay_name) is None
assert mod._endpoints.get(endpoint_name) is None
assert mod._event_bus_policies.get(bus_name) is None
assert mod._connections.get(connection_name) is None
assert mod._api_destinations.get(api_destination_name) is None
assert mod._partner_event_sources.get(partner_key) is None
set_request_account_id("222222222222")
set_request_region("us-west-2")
assert mod._event_buses.get(bus_name) is None
assert mod._rules.get(rule_key) is None
assert mod._targets.get(rule_key) is None
finally:
mod.reset()
set_request_account_id(original_account)
set_request_region(original_region)
# ── iot._jobs / iot._job_executions (tuple execution keys) ─────────────
def test_iot_jobs_and_executions_survive_warm_boot():
"""Jobs are keyed by jobId but executions by a (thingName, jobId) TUPLE —
the on-disk JSON path must round-trip both (tuple keys ride the
repr()/literal_eval leg of the scoped-dict encoder). A drop here would
silently orphan every in-flight OTA rollout on restart."""
mod = _get_module("iot")
mod.reset()
job_id = "persisted-job"
thing = "persisted-thing"
now_ms = 1700000000000
mod._jobs[job_id] = {
"jobId": job_id,
"jobArn": f"arn:aws:iot:us-east-1:000000000000:job/{job_id}",
"description": None,
"targets": [f"arn:aws:iot:us-east-1:000000000000:thing/{thing}"],
"targetSelection": "SNAPSHOT",
"document": "{\"operation\": \"reboot\"}",
"documentSource": None,
"status": "IN_PROGRESS",
"createdAt": now_ms,
"lastUpdatedAt": now_ms,
"completedAt": None,
"presignedUrlConfig": {},
"jobExecutionsRolloutConfig": {},
"snapshotted": True,
}
mod._job_executions[(thing, job_id)] = {
"jobId": job_id,
"thingName": thing,
"status": "IN_PROGRESS",
"statusDetails": {"progress": "42"},
"queuedAt": now_ms,
"startedAt": now_ms,
"lastUpdatedAt": now_ms,
"executionNumber": 1,
"versionNumber": 2,
}
_round_trip_dict(mod, "iot")
job = mod._jobs.get(job_id)
assert job is not None, "_jobs lost across save_state -> load_state"
assert job["status"] == "IN_PROGRESS"
execution = mod._job_executions.get((thing, job_id))
assert execution is not None, (
"_job_executions lost across save_state -> load_state — the "
"(thingName, jobId) tuple key must survive JSON persistence."
)
assert execution["versionNumber"] == 2
assert execution["statusDetails"] == {"progress": "42"}
mod.reset()
def test_eventbridge_legacy_account_scoped_targets_restore_to_rule_region():
"""Legacy target stores are scoped to the EventBridge rule region, not the
target ARN region, when migrating from account-only persistence."""
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
mod = _get_module("eventbridge")
mod.reset()
original_account = get_account_id()
original_region = get_region()
try:
set_request_account_id("111111111111")
set_request_region("us-east-1")
rule_key = mod._rule_key("legacy-rule", "default")
legacy_rules = AccountScopedDict()
legacy_targets = AccountScopedDict()
legacy_rules[rule_key] = {
"Name": "legacy-rule",
"Arn": "arn:aws:events:us-west-2:111111111111:rule/legacy-rule",
"EventBusName": "default",
"ScheduleExpression": "rate(5 minutes)",
"State": "ENABLED",
}
legacy_targets[rule_key] = [
{
"Id": "foreign-sns",
"Arn": "arn:aws:sns:eu-central-1:111111111111:legacy-topic",
}
]
mod.load_persisted_state({"rules": legacy_rules, "targets": legacy_targets})
set_request_region("us-west-2")
assert mod._rules[rule_key]["Name"] == "legacy-rule"
assert mod._targets[rule_key][0]["Id"] == "foreign-sns"
set_request_region("eu-central-1")
assert mod._targets.get(rule_key) is None
finally:
mod.reset()
set_request_account_id(original_account)
set_request_region(original_region)
# ── iot._ca_certificates + iot._registration_codes ────────────────────
def test_iot_ca_registry_and_registration_code_survive_warm_boot():
"""`RegisterCACertificate` writes `_ca_certificates` and
`GetRegistrationCode` writes `_registration_codes`. If either dict is
dropped from get_state()/restore_state(), registered JITR CAs and the
account registration code silently vanish on restart.
The warm boot is spelled out rather than taken from `_round_trip_dict` so
the state is also checked *between* reset() and restore_state(): a dict
missing from reset() survives the round trip untouched, which would satisfy
an after-restore assertion alone while restore_state never restored
anything."""
mod = _get_module("iot")
mod.reset()
ca_id = "c" * 64
ca_arn = f"arn:aws:iot:us-east-1:000000000000:cacert/{ca_id}"
try:
mod._ca_certificates[ca_id] = {
"certificateId": ca_id,
"certificateArn": ca_arn,
"certificatePem": (
"-----BEGIN CERTIFICATE-----\nAA==\n-----END CERTIFICATE-----\n"
),
"status": "ACTIVE",
"autoRegistrationStatus": "ENABLE",
"creationDate": 1700000000,
}
mod._registration_codes["code"] = "d" * 64
persistence.save_state("iot", mod.get_state())
mod.reset()
assert mod._ca_certificates.get(ca_id) is None, (
"_ca_certificates survived reset() — the CA registry must be "
"cleared there, or a restart leaks the previous run's CAs."
)
assert mod._registration_codes.get("code") is None, (
"_registration_codes survived reset() — the registration code must "
"be cleared there."
)
loaded = persistence.load_state("iot")
assert loaded is not None, (
"persistence.load_state('iot') returned None — state file was not "
"written by save_state()."
)
mod.load_persisted_state(loaded)
restored = mod._ca_certificates.get(ca_id)
assert restored is not None, (
"_ca_certificates lost across save_state → load_state — the CA "
"registry must be in both get_state and restore_state."
)
assert restored["certificateArn"] == ca_arn
assert restored["autoRegistrationStatus"] == "ENABLE"
assert mod._registration_codes.get("code") == "d" * 64, (
"_registration_codes lost across save_state → load_state."
)
finally:
mod.reset()
# ── ECS restore-contract regression ──────────────────────────────────
def test_ecs_load_persisted_state_restores_attributes():
"""ECS restores every store through the central loader contract."""
mod = _get_module("ecs")
mod.reset()
arn = "arn:aws:ecs:us-east-1:000000000000:cluster/reload-canary"
mod._clusters[arn] = {"clusterArn": arn, "status": "ACTIVE"}
mod._attributes["i-canary:reload-attr"] = {
"name": "reload-attr",
"value": "v",
"targetType": "container-instance",
"targetId": "i-canary",
}
# Persist via the same path save_all uses on shutdown.
persistence.save_state("ecs", mod.get_state())
mod.reset()
mod.load_persisted_state(persistence.load_state("ecs"))
assert arn in mod._clusters, (
"Cluster lost through the ECS load_persisted_state contract."
)
assert "i-canary:reload-attr" in mod._attributes, (
"ECS _attributes lost through the load_persisted_state contract."
)
mod.reset()
# ── Generic NameError-at-import regression for ALL persisted services ─
def _persisted_services():
"""Return a sorted list of ``(svc_key, mod_name)`` pairs from
``ministack.app._state_map``.
Evaluated by ``@pytest.mark.parametrize(...)`` at test collection
time — `_state_map` is therefore imported when pytest collects this
module, NOT lazily per test case. (Calling it inside the parametrize
decorator means it runs once, at collection.)"""
from ministack.app import _state_map
return sorted(_state_map.items())
@pytest.mark.parametrize("svc_key,mod_name", _persisted_services())
def test_service_defers_persistence_restore_until_registry_loader(svc_key, mod_name):
"""Services must not restore from disk merely because they are imported.
The registry loader is the sole startup restore path. This keeps module
imports lazy and ensures every saved snapshot goes through the uniform
``load_persisted_state(data)`` contract.
"""
spec = importlib.util.find_spec(f"ministack.services.{mod_name}")
assert spec and spec.origin
source = Path(spec.origin).read_text()
tree = ast.parse(source)
import_time_restores = [
node
for node in tree.body
if isinstance(node, (ast.Try, ast.If))
and "load_state(" in (ast.get_source_segment(source, node) or "")
]
assert not import_time_restores, (
f"{mod_name} restores state at import time; restore {svc_key!r} "
"through ministack.app._load_persisted_state instead."
)
assert "from ministack.core.persistence import load_state" not in source
def test_legacy_unwrapped_state_file_loads_and_migrates_region(monkeypatch, tmp_path):
"""U4: a pre-version-stamp on-disk file (no wrapper, account-scoped) must
load through the real load_state() disk path and migrate into a region-
scoped store, recovering region from a value ARN. Prior tests only seeded
dicts in memory, so this disk->migrate path was never exercised."""
import json as _json
from ministack.core.responses import (
AccountRegionScopedDict,
AccountScopedDict,
set_request_region,
)
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
legacy = AccountScopedDict()
legacy._data[("000000000000", "res-1")] = {
"Arn": "arn:aws:appconfig:eu-west-1:000000000000:application/res-1",
}
# Write a legacy (unwrapped, implicit-v1) file exactly as old MiniStack did.
with open(tmp_path / "demo.json", "w") as f:
_json.dump({"store": legacy}, f, default=persistence._json_default)
loaded = persistence.load_state("demo")
assert isinstance(loaded["store"], AccountScopedDict)
region_store = AccountRegionScopedDict()
region_store.update(loaded["store"])
set_request_region("eu-west-1")
assert region_store["res-1"]["Arn"].startswith("arn:aws:appconfig:eu-west-1")
def test_default_state_format_version_stays_v2_and_refuses_newer(monkeypatch, tmp_path):
"""U4: unchanged services stay on v2 and refuse newer state files."""
import json as _json
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
persistence.save_state("sqs", {"k": "v"})
raw = _json.loads((tmp_path / "sqs.json").read_text())
assert raw["__ministack_format__"] == persistence.STATE_FORMAT_VERSION == 2
assert raw["payload"] == {"k": "v"}
assert persistence.load_state("sqs") == {"k": "v"}
(tmp_path / "future.json").write_text(_json.dumps({
"__ministack_format__": persistence.STATE_FORMAT_VERSION + 1,
"payload": {"k": "v"},
}))
assert persistence.load_state("future") is None
@pytest.mark.parametrize(
"service",
[
"elasticache",
"eventbridge",
"firehose",
"kinesis",
"kms",
"mediaconnect",
"pipes",
"scheduler",
"sns",
],
)
def test_region_scoped_backfill_services_stamp_v3_and_reject_v2_reader(
service, monkeypatch, tmp_path
):
"""Services regionalized after v2 must stamp v3 for rollback safety."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
account_id = "123456789012"
region = "us-west-2"
resource_name = f"{service}-regional-resource"
regional_store = AccountRegionScopedDict()
regional_store.set_scoped(
account_id,
region,
resource_name,
{
"Arn": (
f"arn:aws:{service}:{region}:{account_id}:"
f"resource/{resource_name}"
),
"Name": resource_name,
},
)
assert persistence._state_format_version(service) == 3
persistence.save_state(service, {"resources": regional_store})
first_snapshot = _json.loads((tmp_path / f"{service}.json").read_text())
assert first_snapshot["__ministack_format__"] == 3
loaded = persistence.load_state(service)
assert loaded["resources"].get_scoped(
account_id, region, resource_name
)["Name"] == resource_name
persistence.save_state(service, loaded)
second_snapshot = _json.loads((tmp_path / f"{service}.json").read_text())
assert second_snapshot == first_snapshot
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state(service) is None
def test_mediaconnect_is_registered_for_persistence_save():
from ministack.app import _build_persistence_save_dict, _state_map
from ministack.core.responses import (
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import mediaconnect
assert _state_map["mediaconnect"] == "mediaconnect"
original_account = get_account_id()
original_region = get_region()
account_id = "123456789012"
region = "us-west-2"
flow_arn = (
f"arn:aws:mediaconnect:{region}:{account_id}:"
"flow:flow-1234:regional-flow"
)
mediaconnect.reset()
try:
set_request_account_id(account_id)
set_request_region(region)
mediaconnect._flows[flow_arn] = {
"flowArn": flow_arn,
"name": "regional-flow",
}
mediaconnect._tags[flow_arn] = {"Env": "west"}
save_dict = _build_persistence_save_dict()
assert "mediaconnect" in save_dict
state = save_dict["mediaconnect"]()
assert state["flows"].get_scoped(account_id, region, flow_arn)[
"name"
] == "regional-flow"
assert state["tags"][flow_arn] == {"Env": "west"}
finally:
mediaconnect.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_mediaconnect_region_scoped_v3_state_round_trips(monkeypatch, tmp_path):
import json as _json
from ministack.core.responses import (
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import mediaconnect
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
original_account = get_account_id()
original_region = get_region()
account_id = "123456789012"
region = "us-west-2"
other_region = "us-east-1"
flow_arn = (
f"arn:aws:mediaconnect:{region}:{account_id}:"
"flow:flow-1234:regional-flow"
)
mediaconnect.reset()
try:
set_request_account_id(account_id)
set_request_region(region)
mediaconnect._flows[flow_arn] = {
"flowArn": flow_arn,
"name": "regional-flow",
}
mediaconnect._tags[flow_arn] = {"Env": "west"}
persistence.save_state("mediaconnect", mediaconnect.get_state())
first_snapshot = _json.loads(
(tmp_path / "mediaconnect.json").read_text()
)
assert first_snapshot["__ministack_format__"] == 3
mediaconnect.reset()
loaded = persistence.load_state("mediaconnect")
assert loaded is not None
mediaconnect.load_persisted_state(loaded)
set_request_region(region)
assert mediaconnect._flows[flow_arn]["name"] == "regional-flow"
assert mediaconnect._tags[flow_arn] == {"Env": "west"}
set_request_region(other_region)
assert flow_arn not in mediaconnect._flows
persistence.save_state("mediaconnect", mediaconnect.get_state())
second_snapshot = _json.loads(
(tmp_path / "mediaconnect.json").read_text()
)
assert second_snapshot == first_snapshot
finally:
mediaconnect.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_iot_legacy_state_migrates_resource_and_retained_regions():
import base64
from ministack.core.responses import (
AccountScopedDict,
set_request_account_id,
set_request_region,
)
from ministack.services import iot
account_id = "123456789012"
boot_region = "us-east-1"
resource_region = "us-west-2"
resource_name = "legacy-regional"
def legacy_store(value):
store = AccountScopedDict()
store._data[(account_id, resource_name)] = value
return store
legacy_state = {
"things": legacy_store({
"thingName": resource_name,
"thingArn": (
f"arn:aws:iot:{resource_region}:{account_id}:"
f"thing/{resource_name}"
),
}),
"certificates": legacy_store({
"certificateId": resource_name,
"certificateArn": (
f"arn:aws:iot:{resource_region}:{account_id}:"
f"cert/{resource_name}"
),
}),
"policies": legacy_store({
"policyName": resource_name,
"policyArn": (
f"arn:aws:iot:{resource_region}:{account_id}:"
f"policy/{resource_name}"
),
}),
"topic_rules": legacy_store({
"ruleName": resource_name,
"ruleArn": (
f"arn:aws:iot:{resource_region}:{account_id}:"
f"rule/{resource_name}"
),
}),
"mqtt_broker": {
"retained": [{
"topic": f"{account_id}/sensors/temperature",
"payload": base64.b64encode(b"21C").decode("ascii"),
"qos": 1,
}],
},
}
set_request_account_id(account_id)
set_request_region(boot_region)
iot.reset()
iot.broker_reset()
try:
iot.load_persisted_state(legacy_state)
for store in (
iot._things,
iot._certificates,
iot._policies,
iot._topic_rules,
):
assert store.get_scoped(
account_id, resource_region, resource_name
) is not None
assert store.get_scoped(
account_id, boot_region, resource_name
) is None
retained_key = (
f"{account_id}/{boot_region}/sensors/temperature"
)
assert iot._retained[retained_key].payload == b"21C"
assert (
f"{account_id}/{resource_region}/sensors/temperature"
not in iot._retained
)
finally:
iot.reset()
iot.broker_reset()
def test_iot_region_scoped_v3_state_is_idempotent(monkeypatch, tmp_path):
import base64
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
account_id = "123456789012"
region = "us-west-2"
resource_name = "regional-resource"
def regional_store(value):
store = AccountRegionScopedDict()
store.set_scoped(account_id, region, resource_name, value)
return store
state = {
"things": regional_store({
"thingName": resource_name,
"thingArn": (
f"arn:aws:iot:{region}:{account_id}:thing/{resource_name}"
),
}),
"certificates": regional_store({
"certificateId": resource_name,
"certificateArn": (
f"arn:aws:iot:{region}:{account_id}:cert/{resource_name}"
),
}),
"policies": regional_store({
"policyName": resource_name,
"policyArn": (
f"arn:aws:iot:{region}:{account_id}:policy/{resource_name}"
),
}),
"topic_rules": regional_store({
"ruleName": resource_name,
"ruleArn": (
f"arn:aws:iot:{region}:{account_id}:rule/{resource_name}"
),
}),
"mqtt_broker": {
"region_scoped": True,
"retained": [{
"topic": f"{account_id}/{region}/sensors/temperature",
"payload": base64.b64encode(b"21C").decode("ascii"),
"qos": 1,
}],
},
}
persistence.save_state("iot", state)
first_snapshot = _json.loads((tmp_path / "iot.json").read_text())
assert first_snapshot["__ministack_format__"] == 3
loaded = persistence.load_state("iot")
assert loaded is not None
persistence.save_state("iot", loaded)
second_snapshot = _json.loads((tmp_path / "iot.json").read_text())
assert second_snapshot == first_snapshot
def test_iot_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
things = AccountRegionScopedDict()
things.set_scoped(
"123456789012",
"us-west-2",
"regional-thing",
{
"thingName": "regional-thing",
"thingArn": (
"arn:aws:iot:us-west-2:123456789012:"
"thing/regional-thing"
),
},
)
persistence.save_state("iot", {"things": things})
raw = _json.loads((tmp_path / "iot.json").read_text())
assert raw["__ministack_format__"] == 3
loaded = persistence.load_state("iot")["things"]
assert loaded.get_scoped(
"123456789012", "us-west-2", "regional-thing"
)["thingName"] == "regional-thing"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("iot") is None
def test_cloudtrail_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
"""A rollback binary must reject CloudTrail's regional schema instead of
accepting it as v2 and dropping non-boot-region trails."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
trails = AccountRegionScopedDict()
trails.set_scoped(
"000000000000",
"us-west-2",
"regional-trail",
{
"Name": "regional-trail",
"HomeRegion": "us-west-2",
"TrailARN": "arn:aws:cloudtrail:us-west-2:000000000000:trail/regional-trail",
},
)
persistence.save_state("cloudtrail", {"trails": trails})
raw = _json.loads((tmp_path / "cloudtrail.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_trails = persistence.load_state("cloudtrail")["trails"]
assert loaded_trails.get_scoped(
"000000000000", "us-west-2", "regional-trail"
)["HomeRegion"] == "us-west-2"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("cloudtrail") is None
def test_cloudtrail_region_scoped_v3_state_round_trips_idempotently(
monkeypatch, tmp_path
):
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
trails = AccountRegionScopedDict()
trails.set_scoped(
"000000000000",
"us-west-2",
"regional-trail",
{
"Name": "regional-trail",
"HomeRegion": "us-west-2",
"TrailARN": "arn:aws:cloudtrail:us-west-2:000000000000:trail/regional-trail",
},
)
selectors = AccountRegionScopedDict()
selectors.set_scoped(
"000000000000",
"us-west-2",
"regional-trail",
[{"ReadWriteType": "All", "IncludeManagementEvents": True}],
)
persistence.save_state(
"cloudtrail",
{"trails": trails, "event_selectors": selectors},
)
first_snapshot = _json.loads((tmp_path / "cloudtrail.json").read_text())
loaded = persistence.load_state("cloudtrail")
assert loaded["trails"].get_scoped(
"000000000000", "us-west-2", "regional-trail"
)["TrailARN"].endswith("trail/regional-trail")
assert loaded["event_selectors"].get_scoped(
"000000000000", "us-west-2", "regional-trail"
)[0]["ReadWriteType"] == "All"
persistence.save_state("cloudtrail", loaded)
second_snapshot = _json.loads((tmp_path / "cloudtrail.json").read_text())
assert second_snapshot == first_snapshot
def test_cloudtrail_legacy_state_restores_trails_by_home_region():
from ministack.core.responses import (
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import cloudtrail
original_account = get_account_id()
original_region = get_region()
account_id = "111111111111"
boot_region = "us-east-1"
west_region = "us-west-2"
west_name = "legacy-west-trail"
east_name = "legacy-east-trail"
orphan_name = "legacy-orphan-trail"
west_arn = f"arn:aws:cloudtrail:{west_region}:{account_id}:trail/{west_name}"
try:
set_request_account_id(account_id)
set_request_region(boot_region)
cloudtrail.reset()
cloudtrail.load_persisted_state(
{
"trails": {
west_name: {
"Name": west_name,
"HomeRegion": west_region,
"TrailARN": west_arn,
},
east_name: {
"Name": east_name,
"HomeRegion": boot_region,
"TrailARN": (
f"arn:aws:cloudtrail:{boot_region}:{account_id}:"
f"trail/{east_name}"
),
},
},
"event_selectors": {
west_name: [{"ReadWriteType": "All"}],
orphan_name: [{"ReadWriteType": "WriteOnly"}],
},
"trail_tags": {west_arn: {"env": "legacy"}},
}
)
assert cloudtrail._trails.get_scoped(account_id, west_region, west_name)[
"HomeRegion"
] == west_region
assert cloudtrail._event_selectors.get_scoped(
account_id, west_region, west_name
) == [{"ReadWriteType": "All"}]
assert cloudtrail._trails.get_scoped(account_id, boot_region, west_name) is None
assert cloudtrail._event_selectors.get_scoped(
account_id, boot_region, orphan_name
) == [{"ReadWriteType": "WriteOnly"}]
assert cloudtrail._trail_tags.get_scoped(
account_id, boot_region, west_arn
) == {"env": "legacy"}
finally:
cloudtrail.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_cloudtrail_persistence_lifecycle_restores_all_regional_accounts(
monkeypatch, tmp_path
):
from ministack.app import _build_persistence_save_dict, _state_map
from ministack.core.responses import set_request_account_id, set_request_region
from ministack.services import cloudtrail
boot_region = "us-east-1"
west_region = "us-west-2"
first_account = "111111111111"
second_account = "222222222222"
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
set_request_account_id(first_account)
set_request_region(boot_region)
cloudtrail.reset()
try:
cloudtrail._trails.set_scoped(
first_account,
west_region,
"acct-one-west",
{
"Name": "acct-one-west",
"HomeRegion": west_region,
"TrailARN": (
f"arn:aws:cloudtrail:{west_region}:{first_account}:"
"trail/acct-one-west"
),
},
)
cloudtrail._trails.set_scoped(
second_account,
boot_region,
"acct-two-east",
{
"Name": "acct-two-east",
"HomeRegion": boot_region,
"TrailARN": (
f"arn:aws:cloudtrail:{boot_region}:{second_account}:"
"trail/acct-two-east"
),
},
)
assert _state_map["cloudtrail"] == "cloudtrail"
save_dict = _build_persistence_save_dict()
persistence.save_all({"cloudtrail": save_dict["cloudtrail"]})
cloudtrail.reset()
cloudtrail.load_persisted_state(persistence.load_state("cloudtrail"))
assert cloudtrail._trails.get_scoped(
first_account, west_region, "acct-one-west"
)["HomeRegion"] == west_region
assert cloudtrail._trails.get_scoped(
second_account, boot_region, "acct-two-east"
)["HomeRegion"] == boot_region
assert cloudtrail._trails.get_scoped(
first_account, boot_region, "acct-one-west"
) is None
finally:
cloudtrail.reset()
def test_ecs_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
"""A rollback binary must reject ECS's regional schema instead of
accepting it as v2 and silently dropping every regional store."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
clusters = AccountRegionScopedDict()
clusters.set_scoped(
"000000000000",
"us-west-2",
"regional-cluster",
{"clusterArn": "arn:aws:ecs:us-west-2:000000000000:cluster/regional-cluster"},
)
persistence.save_state("ecs", {"clusters": clusters})
raw = _json.loads((tmp_path / "ecs.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_clusters = persistence.load_state("ecs")["clusters"]
assert loaded_clusters.get_scoped(
"000000000000", "us-west-2", "regional-cluster"
)["clusterArn"].endswith("cluster/regional-cluster")
# Simulate the previous binary, whose highest understood format is v2.
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("ecs") is None
def test_ecr_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
"""A rollback binary must reject ECR's regional schema instead of
accepting it as v2 and silently dropping every regional store."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
repositories = AccountRegionScopedDict()
repositories.set_scoped(
"000000000000",
"us-west-2",
"regional-repository",
{
"repositoryArn": (
"arn:aws:ecr:us-west-2:000000000000:"
"repository/regional-repository"
)
},
)
persistence.save_state("ecr", {"repositories": repositories})
raw = _json.loads((tmp_path / "ecr.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_repositories = persistence.load_state("ecr")["repositories"]
assert loaded_repositories.get_scoped(
"000000000000", "us-west-2", "regional-repository"
)["repositoryArn"].endswith("repository/regional-repository")
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("ecr") is None
def test_appsync_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
"""A rollback binary must reject AppSync's regional schema instead of
accepting it as v2 and silently dropping every regional store."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
apis = AccountRegionScopedDict()
apis.set_scoped(
"000000000000",
"us-west-2",
"regional-api",
{
"apiId": "regional-api",
"arn": "arn:aws:appsync:us-west-2:000000000000:apis/regional-api",
},
)
persistence.save_state("appsync", {"apis": apis})
raw = _json.loads((tmp_path / "appsync.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_apis = persistence.load_state("appsync")["apis"]
assert loaded_apis.get_scoped(
"000000000000", "us-west-2", "regional-api"
)["apiId"] == "regional-api"
# Simulate the previous binary, whose highest understood format is v2.
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("appsync") is None
def test_appsync_events_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
"""A rollback binary must reject AppSync Events' regional schema instead
of accepting it as v2 and silently dropping APIs, namespaces, and keys."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
apis = AccountRegionScopedDict()
apis.set_scoped(
"000000000000",
"us-west-2",
"regional-events-api",
{
"apiArn": (
"arn:aws:appsync:us-west-2:000000000000:"
"apis/regional-events-api"
),
},
)
persistence.save_state("appsync_events", {"apis": apis})
raw = _json.loads((tmp_path / "appsync_events.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_apis = persistence.load_state("appsync_events")["apis"]
assert loaded_apis.get_scoped(
"000000000000", "us-west-2", "regional-events-api"
)["apiArn"].endswith("apis/regional-events-api")
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("appsync_events") is None
def test_eks_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
"""A rollback binary must reject EKS regional state instead of dropping it."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
clusters = AccountRegionScopedDict()
clusters.set_scoped(
"000000000000",
"us-west-2",
"regional-cluster",
{
"name": "regional-cluster",
"arn": (
"arn:aws:eks:us-west-2:000000000000:"
"cluster/regional-cluster"
),
},
)
persistence.save_state("eks", {"clusters": clusters})
raw = _json.loads((tmp_path / "eks.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_clusters = persistence.load_state("eks")["clusters"]
assert loaded_clusters.get_scoped(
"000000000000", "us-west-2", "regional-cluster"
)["arn"].endswith("cluster/regional-cluster")
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("eks") is None
def test_emr_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
"""A rollback binary must reject EMR regional state instead of dropping it."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
clusters = AccountRegionScopedDict()
clusters.set_scoped(
"000000000000",
"us-west-2",
"j-LEGACYCLUSTER",
{
"Id": "j-LEGACYCLUSTER",
"ClusterArn": (
"arn:aws:elasticmapreduce:us-west-2:000000000000:"
"cluster/j-LEGACYCLUSTER"
),
},
)
persistence.save_state("emr", {"_clusters": clusters})
raw = _json.loads((tmp_path / "emr.json").read_text())
assert raw["__ministack_format__"] == 3
loaded = persistence.load_state("emr")["_clusters"]
assert loaded.get_scoped(
"000000000000", "us-west-2", "j-LEGACYCLUSTER"
)["Id"] == "j-LEGACYCLUSTER"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("emr") is None
def test_transfer_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
"""A rollback binary must reject Transfer's regional schema instead of
accepting it as v2 and silently dropping regional servers and users."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
servers = AccountRegionScopedDict()
servers.set_scoped(
"000000000000",
"us-west-2",
"s-regionalserver01",
{
"ServerId": "s-regionalserver01",
"Arn": (
"arn:aws:transfer:us-west-2:000000000000:"
"server/s-regionalserver01"
),
},
)
users = AccountRegionScopedDict()
users.set_scoped(
"000000000000",
"us-west-2",
"s-regionalserver01/regional-user",
{
"ServerId": "s-regionalserver01",
"UserName": "regional-user",
"Arn": (
"arn:aws:transfer:us-west-2:000000000000:"
"user/s-regionalserver01/regional-user"
),
},
)
persistence.save_state("transfer", {"servers": servers, "users": users})
raw = _json.loads((tmp_path / "transfer.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_servers = persistence.load_state("transfer")["servers"]
assert loaded_servers.get_scoped(
"000000000000", "us-west-2", "s-regionalserver01"
)["ServerId"] == "s-regionalserver01"
loaded_users = persistence.load_state("transfer")["users"]
assert loaded_users.get_scoped(
"000000000000",
"us-west-2",
"s-regionalserver01/regional-user",
)["UserName"] == "regional-user"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("transfer") is None
def test_resource_groups_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
"""A rollback binary must reject Resource Groups' regional schema instead
of accepting it as v2 and silently dropping every regional store."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
groups = AccountRegionScopedDict()
groups.set_scoped(
"000000000000",
"us-west-2",
"regional-group",
{
"GroupArn": (
"arn:aws:resource-groups:us-west-2:000000000000:"
"group/regional-group"
)
},
)
persistence.save_state("resource_groups", {"groups": groups})
raw = _json.loads((tmp_path / "resource_groups.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_groups = persistence.load_state("resource_groups")["groups"]
assert loaded_groups.get_scoped(
"000000000000", "us-west-2", "regional-group"
)["GroupArn"].endswith("group/regional-group")
# Simulate the previous binary, whose highest understood format is v2.
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("resource_groups") is None
def test_codebuild_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
"""A rollback binary must reject CodeBuild's regional schema instead of
accepting it as v2 and silently dropping every regional store."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
projects = AccountRegionScopedDict()
projects.set_scoped(
"000000000000",
"us-west-2",
"regional-project",
{
"arn": (
"arn:aws:codebuild:us-west-2:000000000000:"
"project/regional-project"
)
},
)
persistence.save_state("codebuild", {"projects": projects})
raw = _json.loads((tmp_path / "codebuild.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_projects = persistence.load_state("codebuild")["projects"]
assert loaded_projects.get_scoped(
"000000000000", "us-west-2", "regional-project"
)["arn"].endswith("project/regional-project")
# Simulate the previous binary, whose highest understood format is v2.
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("codebuild") is None
def test_mq_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
"""A rollback binary must reject MQ's regional schema instead of
accepting it as v2 and silently dropping every regional store."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
brokers = AccountRegionScopedDict()
brokers.set_scoped(
"000000000000",
"us-west-2",
"regional-broker",
{
"brokerArn": (
"arn:aws:mq:us-west-2:000000000000:broker:regional-broker"
)
},
)
persistence.save_state("mq", {"brokers": brokers})
raw = _json.loads((tmp_path / "mq.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_brokers = persistence.load_state("mq")["brokers"]
assert loaded_brokers.get_scoped(
"000000000000", "us-west-2", "regional-broker"
)["brokerArn"].endswith("broker:regional-broker")
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("mq") is None
def test_mwaa_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
"""A rollback binary must reject MWAA's regional schema instead of
accepting it as v2 and silently dropping every regional environment."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
environments = AccountRegionScopedDict()
environments.set_scoped(
"000000000000",
"us-west-2",
"regional-environment",
{
"Name": "regional-environment",
"Arn": (
"arn:aws:airflow:us-west-2:000000000000:"
"environment/regional-environment"
),
},
)
persistence.save_state("mwaa", {"environments": environments})
raw = _json.loads((tmp_path / "mwaa.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_environments = persistence.load_state("mwaa")["environments"]
assert loaded_environments.get_scoped(
"000000000000", "us-west-2", "regional-environment"
)["Name"] == "regional-environment"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("mwaa") is None
def test_ses_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
"""A rollback binary must reject both SES persistence files after their
stores become regional instead of accepting v2 and restoring them empty."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
for service in ("ses", "ses_v2"):
identities = AccountRegionScopedDict()
identities.set_scoped(
"000000000000",
"us-west-2",
"[email protected]",
{"VerificationStatus": "Success"},
)
persistence.save_state(service, {"_identities": identities})
raw = _json.loads((tmp_path / f"{service}.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_identities = persistence.load_state(service)["_identities"]
assert loaded_identities.get_scoped(
"000000000000", "us-west-2", "[email protected]"
)["VerificationStatus"] == "Success"
# Simulate the previous binary, whose highest understood format is v2.
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("ses") is None
assert persistence.load_state("ses_v2") is None
def test_batch_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
"""A rollback binary must reject Batch's regional schema instead of
accepting it as v2 and silently dropping every regional store."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
jobs = AccountRegionScopedDict()
jobs.set_scoped(
"000000000000",
"us-west-2",
"regional-job",
{"jobArn": "arn:aws:batch:us-west-2:000000000000:job/regional-job"},
)
persistence.save_state("batch", {"jobs": jobs})
raw = _json.loads((tmp_path / "batch.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_jobs = persistence.load_state("batch")["jobs"]
assert loaded_jobs.get_scoped(
"000000000000", "us-west-2", "regional-job"
)["jobArn"].endswith("job/regional-job")
# Simulate the previous binary, whose highest understood format is v2.
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("batch") is None
def test_autoscaling_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
"""A rollback binary must reject Auto Scaling's regional schema instead
of accepting it as v2 and silently dropping every regional store."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
asgs = AccountRegionScopedDict()
asgs.set_scoped(
"000000000000",
"us-west-2",
"regional-asg",
{
"AutoScalingGroupARN": (
"arn:aws:autoscaling:us-west-2:000000000000:"
"autoScalingGroup:regional:autoScalingGroupName/regional-asg"
)
},
)
persistence.save_state("autoscaling", {"asgs": asgs})
raw = _json.loads((tmp_path / "autoscaling.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_asgs = persistence.load_state("autoscaling")["asgs"]
assert loaded_asgs.get_scoped(
"000000000000", "us-west-2", "regional-asg"
)["AutoScalingGroupARN"].endswith("autoScalingGroupName/regional-asg")
# Simulate the previous binary, whose highest understood format is v2.
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("autoscaling") is None
def test_athena_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
"""A rollback binary must reject Athena's regional schema instead of
accepting it as v2 and silently dropping every regional store."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
workgroups = AccountRegionScopedDict()
workgroups.set_scoped(
"000000000000",
"us-west-2",
"regional-workgroup",
{"Name": "regional-workgroup", "Description": "west"},
)
persistence.save_state("athena", {"_workgroups": workgroups})
raw = _json.loads((tmp_path / "athena.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_workgroups = persistence.load_state("athena")["_workgroups"]
assert loaded_workgroups.get_scoped(
"000000000000", "us-west-2", "regional-workgroup"
)["Description"] == "west"
# Simulate the previous binary, whose highest understood format is v2.
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("athena") is None
def test_acm_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
"""A rollback binary must reject ACM's regional schema instead of accepting
it as v2 and silently dropping every regional certificate store."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
certificates = AccountRegionScopedDict()
arn = "arn:aws:acm:us-west-2:000000000000:certificate/regional-cert"
certificates.set_scoped(
"000000000000", "us-west-2", arn,
{"CertificateArn": arn, "DomainName": "west.example.com"},
)
persistence.save_state("acm", {"_certificates": certificates})
raw = _json.loads((tmp_path / "acm.json").read_text())
assert raw["__ministack_format__"] == 3
loaded = persistence.load_state("acm")["_certificates"]
assert loaded.get_scoped(
"000000000000", "us-west-2", arn
)["DomainName"] == "west.example.com"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("acm") is None
def test_alb_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
"""A rollback binary must reject ALB's regional schema instead of accepting
it as v2 and silently dropping every regional load-balancer store."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
lbs = AccountRegionScopedDict()
arn = "arn:aws:elasticloadbalancing:us-west-2:000000000000:loadbalancer/app/west/1"
lbs.set_scoped(
"000000000000", "us-west-2", arn,
{"LoadBalancerArn": arn, "LoadBalancerName": "west"},
)
persistence.save_state("alb", {"_lbs": lbs})
raw = _json.loads((tmp_path / "alb.json").read_text())
assert raw["__ministack_format__"] == 3
loaded = persistence.load_state("alb")["_lbs"]
assert loaded.get_scoped(
"000000000000", "us-west-2", arn
)["LoadBalancerName"] == "west"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("alb") is None
def test_bedrock_agentcore_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
"""A rollback binary must reject AgentCore's regional schema instead of
accepting it as v2 and silently dropping every regional store."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
runtimes = AccountRegionScopedDict()
runtimes.set_scoped(
"000000000000",
"us-west-2",
"regional-runtime",
{
"agentRuntimeId": "regional-runtime",
"agentRuntimeArn": (
"arn:aws:bedrock-agentcore:us-west-2:000000000000:"
"agent/11111111-1111-1111-1111-111111111111:1"
),
},
)
persistence.save_state(
"bedrock_agentcore",
{"runtimes": runtimes, "endpoints": AccountRegionScopedDict()},
)
raw = _json.loads((tmp_path / "bedrock_agentcore.json").read_text())
assert raw["__ministack_format__"] == 3
loaded = persistence.load_state("bedrock_agentcore")["runtimes"]
assert loaded.get_scoped(
"000000000000", "us-west-2", "regional-runtime"
)["agentRuntimeArn"].endswith(":1")
# Simulate the previous binary, whose highest understood format is v2.
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("bedrock_agentcore") is None
def test_inspector2_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
"""A rollback binary must reject Inspector2 regional account buckets."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
configs = AccountRegionScopedDict()
configs.set_scoped(
"000000000000",
"us-west-2",
"000000000000",
{"ecr": {"status": "ENABLED"}},
)
persistence.save_state("inspector2", {"account_config": configs})
raw = _json.loads((tmp_path / "inspector2.json").read_text())
assert raw["__ministack_format__"] == 3
loaded = persistence.load_state("inspector2")["account_config"]
assert loaded.get_scoped(
"000000000000", "us-west-2", "000000000000"
) == {"ecr": {"status": "ENABLED"}}
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("inspector2") is None
def test_efs_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
"""A rollback binary must reject EFS regional state instead of dropping it."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
file_systems = AccountRegionScopedDict()
file_systems.set_scoped(
"000000000000",
"us-west-2",
"fs-11111111111111111",
{
"FileSystemId": "fs-11111111111111111",
"FileSystemArn": (
"arn:aws:elasticfilesystem:us-west-2:000000000000:"
"file-system/fs-11111111111111111"
),
},
)
persistence.save_state("efs", {"file_systems": file_systems})
raw = _json.loads((tmp_path / "efs.json").read_text())
assert raw["__ministack_format__"] == 3
loaded = persistence.load_state("efs")["file_systems"]
assert loaded.get_scoped(
"000000000000", "us-west-2", "fs-11111111111111111"
)["FileSystemId"] == "fs-11111111111111111"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("efs") is None
def test_s3files_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
"""A rollback binary must reject S3 Files' regional schema instead of
accepting it as v2 and silently dropping every regional store."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
file_systems = AccountRegionScopedDict()
file_systems.set_scoped(
"000000000000",
"us-west-2",
"fs-11111111111111111",
{
"fileSystemId": "fs-11111111111111111",
"fileSystemArn": (
"arn:aws:s3files:us-west-2:000000000000:"
"file-system/fs-11111111111111111"
),
},
)
persistence.save_state("s3files", {"file_systems": file_systems})
raw = _json.loads((tmp_path / "s3files.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_file_systems = persistence.load_state("s3files")["file_systems"]
assert loaded_file_systems.get_scoped(
"000000000000", "us-west-2", "fs-11111111111111111"
)["fileSystemId"] == "fs-11111111111111111"
# Simulate the previous binary, whose highest understood format is v2.
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("s3files") is None
def test_ec2_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
"""A rollback binary must reject EC2 regional state instead of accepting
account-only stores that would collapse same-name resources across regions."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
key_pairs = AccountRegionScopedDict()
key_pairs.set_scoped(
"000000000000",
"us-west-2",
"same-key",
{"KeyName": "same-key", "KeyPairId": "key-west"},
)
persistence.save_state("ec2", {"key_pairs": key_pairs})
raw = _json.loads((tmp_path / "ec2.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_key_pairs = persistence.load_state("ec2")["key_pairs"]
assert loaded_key_pairs.get_scoped(
"000000000000", "us-west-2", "same-key"
)["KeyPairId"] == "key-west"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("ec2") is None
def test_waf_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
"""A rollback binary must reject WAF's regional schema instead of accepting
account-only stores that would collapse REGIONAL and CLOUDFRONT scopes."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
web_acls = AccountRegionScopedDict()
web_acls.set_scoped(
"000000000000",
"us-west-2",
"regional-acl",
{
"ARN": (
"arn:aws:wafv2:us-west-2:000000000000:"
"regional/webacl/regional-acl/regional-acl"
),
"Id": "regional-acl",
"Name": "regional-acl",
"Scope": "REGIONAL",
},
)
persistence.save_state("waf", {"_web_acls": web_acls})
raw = _json.loads((tmp_path / "waf.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_web_acls = persistence.load_state("waf")["_web_acls"]
assert loaded_web_acls.get_scoped(
"000000000000",
"us-west-2",
"regional-acl",
)["Name"] == "regional-acl"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("waf") is None
def test_cognito_region_scoped_state_round_trips_and_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
import json as _json
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import cognito
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
original_account = get_account_id()
original_region = get_region()
account = "111111111111"
pool_id = "us-west-2_roundtrip"
identity_pool_id = "us-west-2:00000000-0000-4000-8000-000000000002"
def _legacy_store(key, value):
store = AccountScopedDict()
store._data[(account, key)] = value
return store
legacy_payload = {
"user_pools": _legacy_store(
pool_id, {"Id": pool_id, "Name": "roundtrip"}
),
"pool_domain_map": _legacy_store("roundtrip-domain", pool_id),
"identity_pools": _legacy_store(
identity_pool_id,
{
"IdentityPoolId": identity_pool_id,
"IdentityPoolName": "roundtrip",
},
),
"identity_tags": _legacy_store(identity_pool_id, {"scope": "west"}),
}
(tmp_path / "cognito.json").write_text(
_json.dumps(
{"__ministack_format__": 2, "payload": legacy_payload},
default=persistence._json_default,
)
)
try:
set_request_account_id(account)
set_request_region("us-east-1")
cognito.reset()
cognito.load_persisted_state(persistence.load_state("cognito"))
assert cognito._user_pools.get_scoped(
account, "us-west-2", pool_id
)["Name"] == "roundtrip"
assert cognito._user_pools.get_scoped(
account, "us-east-1", pool_id
) is None
assert cognito._pool_domain_map.get_scoped(
account, "us-west-2", "roundtrip-domain"
) == pool_id
assert cognito._identity_pools.get_scoped(
account, "us-west-2", identity_pool_id
)["IdentityPoolName"] == "roundtrip"
assert cognito._identity_tags.get_scoped(
account, "us-west-2", identity_pool_id
) == {"scope": "west"}
persistence.save_state("cognito", cognito.get_state())
raw = _json.loads((tmp_path / "cognito.json").read_text())
assert raw["__ministack_format__"] == 3
loaded = persistence.load_state("cognito")["user_pools"]
assert loaded.get_scoped(account, "us-west-2", pool_id)[
"Name"
] == "roundtrip"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("cognito") is None
finally:
cognito.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_cognito_restore_rebuilds_schema_attributes_from_legacy_snapshot(monkeypatch):
"""A pool snapshotted before SchemaAttributes existed still describes fully."""
from ministack.core.responses import (
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import cognito
original_account = get_account_id()
original_region = get_region()
pool_id = "us-west-2_legacyschema"
try:
set_request_account_id("000000000000")
set_request_region("us-west-2")
cognito.reset()
cognito.load_persisted_state({
"user_pools": {
pool_id: {
"Id": pool_id,
"Name": "legacy",
"Schema": [{"Name": "tenant", "AttributeDataType": "String"}],
}
}
})
pool = cognito._pool_out(cognito._user_pools[pool_id])
names = [a["Name"] for a in pool["SchemaAttributes"]]
assert "sub" in names and "email" in names
assert "custom:tenant" in names
# The request-shaped key does not survive into the response shape.
assert "Schema" not in pool
finally:
cognito.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_waf_region_scoped_v3_state_round_trips_idempotently(monkeypatch, tmp_path):
import json as _json
from ministack.core.responses import AccountRegionScopedDict, AccountScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
web_acls = AccountRegionScopedDict()
web_acls.set_scoped(
"000000000000",
"us-east-1",
"global-acl",
{
"ARN": (
"arn:aws:wafv2:us-east-1:000000000000:"
"global/webacl/global-acl/global-acl"
),
"Id": "global-acl",
"Name": "global-acl",
"Scope": "CLOUDFRONT",
},
)
associations = AccountRegionScopedDict()
associations.set_scoped(
"000000000000",
"us-west-2",
"arn:aws:elasticloadbalancing:us-west-2:000000000000:loadbalancer/app/app/id",
(
"arn:aws:wafv2:us-east-1:000000000000:"
"global/webacl/global-acl/global-acl"
),
)
tags = AccountScopedDict()
tags.set_scoped(
"000000000000",
"us-west-2",
(
"arn:aws:wafv2:us-east-1:000000000000:"
"global/webacl/global-acl/global-acl"
),
[{"Key": "scope", "Value": "global"}],
)
persistence.save_state(
"waf",
{
"_web_acls": web_acls,
"_associations": associations,
"_waf_tags": tags,
},
)
first_snapshot = _json.loads((tmp_path / "waf.json").read_text())
loaded = persistence.load_state("waf")
assert loaded["_web_acls"].get_scoped(
"000000000000",
"us-east-1",
"global-acl",
)["Scope"] == "CLOUDFRONT"
assert loaded["_associations"].get_scoped(
"000000000000",
"us-west-2",
"arn:aws:elasticloadbalancing:us-west-2:000000000000:loadbalancer/app/app/id",
).endswith("global/webacl/global-acl/global-acl")
persistence.save_state("waf", loaded)
second_snapshot = _json.loads((tmp_path / "waf.json").read_text())
assert second_snapshot == first_snapshot
def test_ec2_legacy_state_restores_to_boot_region_without_arn_mining():
"""Legacy EC2 state is one coherent graph even when records embed foreign ARNs."""
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import ec2
original_account = get_account_id()
original_region = get_region()
account_id = "123456789012"
boot_region = "us-east-1"
foreign_region = "us-west-2"
vpcs = AccountScopedDict()
flow_logs = AccountScopedDict()
try:
set_request_account_id(account_id)
set_request_region(boot_region)
ec2.reset()
vpcs["vpc-legacy"] = {
"VpcId": "vpc-legacy",
"CidrBlock": "10.70.0.0/16",
"OwnerId": account_id,
}
flow_logs["fl-legacy"] = {
"FlowLogId": "fl-legacy",
"ResourceId": "vpc-legacy",
"DeliverLogsPermissionArn": (
f"arn:aws:logs:{foreign_region}:{account_id}:log-group:foreign"
),
"LogDestination": f"arn:aws:logs:{foreign_region}:{account_id}:log-group:foreign",
}
ec2.load_persisted_state({"vpcs": vpcs, "flow_logs": flow_logs})
assert ec2._vpcs.get_scoped(account_id, boot_region, "vpc-legacy")["VpcId"] == "vpc-legacy"
assert ec2._flow_logs.get_scoped(account_id, boot_region, "fl-legacy")["ResourceId"] == "vpc-legacy"
assert ec2._flow_logs.get_scoped(account_id, foreign_region, "fl-legacy") is None
finally:
ec2.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_ec2_legacy_vpc_peering_restores_to_boot_region_graph():
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import ec2
original_account = get_account_id()
original_region = get_region()
account_id = "123456789012"
boot_region = "us-east-1"
peer_region = "us-west-2"
pcx_id = "pcx-legacy000000001"
vpcs = AccountScopedDict()
peerings = AccountScopedDict()
tags = AccountScopedDict()
try:
set_request_account_id(account_id)
set_request_region(boot_region)
ec2.reset()
vpcs["vpc-east"] = {
"VpcId": "vpc-east",
"CidrBlock": "10.0.0.0/16",
"State": "available",
"IsDefault": False,
"OwnerId": account_id,
}
vpcs["vpc-west"] = {
"VpcId": "vpc-west",
"CidrBlock": "10.1.0.0/16",
"State": "available",
"IsDefault": False,
"OwnerId": account_id,
}
peerings[pcx_id] = {
"VpcPeeringConnectionId": pcx_id,
"RequesterVpcInfo": {
"VpcId": "vpc-east",
"OwnerId": account_id,
"Region": boot_region,
},
"AccepterVpcInfo": {
"VpcId": "vpc-west",
"OwnerId": account_id,
"Region": peer_region,
},
"Status": {
"Code": "pending-acceptance",
"Message": "Pending Acceptance",
},
}
tags[pcx_id] = [{"Key": "Scope", "Value": "legacy"}]
ec2.load_persisted_state({"vpcs": vpcs, "vpc_peering": peerings, "tags": tags})
boot_record = ec2._vpc_peering.get_scoped(account_id, boot_region, pcx_id)
peer_record = ec2._vpc_peering.get_scoped(account_id, peer_region, pcx_id)
assert boot_record["VpcPeeringConnectionId"] == pcx_id
assert boot_record["RequesterVpcInfo"]["Region"] == boot_region
assert boot_record["AccepterVpcInfo"]["Region"] == boot_region
assert peer_record is None
assert ec2._vpcs.get_scoped(account_id, boot_region, "vpc-east") is not None
assert ec2._vpcs.get_scoped(account_id, boot_region, "vpc-west") is not None
assert ec2._vpcs.get_scoped(account_id, peer_region, "vpc-west") is None
assert ec2._tags.get_scoped(account_id, boot_region, pcx_id) == tags[pcx_id]
assert ec2._tags.get_scoped(account_id, peer_region, pcx_id) is None
assert (account_id, peer_region) not in ec2._default_initialized_scopes
set_request_region(peer_region)
ec2._ensure_defaults_initialized()
assert (
ec2._vpcs.get_scoped(account_id, peer_region, ec2._DEFAULT_VPC_ID)
is not None
)
ec2._set_vpc_peering_status(boot_record, "active", "Active")
assert (
ec2._vpc_peering.get_scoped(account_id, boot_region, pcx_id)["Status"]["Code"]
== "active"
)
assert ec2._vpc_peering.get_scoped(account_id, peer_region, pcx_id) is None
finally:
ec2.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_ec2_restore_preserves_deleted_default_resource_scope():
from ministack.core.responses import (
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import ec2
original_account = get_account_id()
original_region = get_region()
account_id = "123456789012"
region = "us-west-2"
try:
set_request_account_id(account_id)
set_request_region(region)
ec2.reset()
for store in (
ec2._vpcs,
ec2._subnets,
ec2._security_groups,
ec2._network_acls,
ec2._internet_gateways,
ec2._route_tables,
):
store.clear()
state = ec2.get_state()
assert {"AccountId": account_id, "Region": region} in state[
"default_initialized_scopes"
]
ec2.load_persisted_state(state)
ec2._ensure_defaults_initialized()
assert ec2._vpcs.get_scoped(account_id, region, ec2._DEFAULT_VPC_ID) is None
assert (
ec2._subnets.get_scoped(account_id, region, ec2._DEFAULT_SUBNET_ID)
is None
)
assert (
ec2._security_groups.get_scoped(account_id, region, ec2._DEFAULT_SG_ID)
is None
)
finally:
set_request_account_id(original_account)
set_request_region(original_region)
ec2.reset()
def test_ec2_legacy_generated_default_vpc_marks_scope_initialized():
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import ec2
original_account = get_account_id()
original_region = get_region()
account_id = "123456789012"
region = "us-west-2"
generated_default_vpc_id = "vpc-0abc1234def567890"
vpcs = AccountScopedDict()
try:
set_request_account_id(account_id)
set_request_region(region)
ec2.reset()
vpcs[generated_default_vpc_id] = {
"VpcId": generated_default_vpc_id,
"CidrBlock": "172.31.0.0/16",
"State": "available",
"IsDefault": True,
"DhcpOptionsId": "dopt-00000001",
"InstanceTenancy": "default",
"OwnerId": account_id,
"DefaultNetworkAclId": "acl-0abc1234def567890",
"DefaultSecurityGroupId": "sg-0abc1234def567890",
"MainRouteTableId": "rtb-0abc1234def567890",
}
ec2.load_persisted_state({"vpcs": vpcs})
ec2._ensure_defaults_initialized()
assert (
ec2._vpcs.get_scoped(account_id, region, generated_default_vpc_id)
is not None
)
assert ec2._vpcs.get_scoped(account_id, region, ec2._DEFAULT_VPC_ID) is None
finally:
set_request_account_id(original_account)
set_request_region(original_region)
ec2.reset()
def test_ec2_region_scoped_v3_state_is_idempotent(monkeypatch, tmp_path):
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
account_id = "123456789012"
region = "us-west-2"
key_pairs = AccountRegionScopedDict()
key_pairs.set_scoped(
account_id,
region,
"regional-key",
{"KeyName": "regional-key", "KeyPairId": "key-west"},
)
flow_logs = AccountRegionScopedDict()
flow_logs.set_scoped(
account_id,
region,
"fl-regional",
{
"FlowLogId": "fl-regional",
"ResourceId": "vpc-regional",
"LogDestination": f"arn:aws:logs:{region}:{account_id}:log-group:regional",
},
)
persistence.save_state("ec2", {"key_pairs": key_pairs, "flow_logs": flow_logs})
first_snapshot = _json.loads((tmp_path / "ec2.json").read_text())
assert first_snapshot["__ministack_format__"] == 3
assert f"{account_id}\x00{region}\x00'regional-key'" in first_snapshot["payload"]["key_pairs"]["data"]
loaded = persistence.load_state("ec2")
assert loaded["key_pairs"].get_scoped(account_id, region, "regional-key")["KeyPairId"] == "key-west"
persistence.save_state("ec2", loaded)
second_snapshot = _json.loads((tmp_path / "ec2.json").read_text())
assert second_snapshot == first_snapshot
def test_opensearch_region_scoped_v3_state_is_idempotent(monkeypatch, tmp_path):
import json as _json
from ministack.core.responses import AccountRegionScopedDict, AccountScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
account_id = "123456789012"
region = "us-west-2"
domain_name = "regional-domain"
package_id = "FPKG123456"
association_key = f"{package_id}:{domain_name}"
domain_arn = f"arn:aws:es:{region}:{account_id}:domain/{domain_name}"
def regional_store(key, value):
store = AccountRegionScopedDict()
store.set_scoped(account_id, region, key, value)
return store
tags = AccountScopedDict()
tags._data[(account_id, domain_arn)] = [{"Key": "Env", "Value": "west"}]
state = {
"domains": regional_store(
domain_name,
{"DomainName": domain_name, "ARN": domain_arn},
),
"tags": tags,
"change_progress": regional_store(
domain_name,
{"ChangeId": "change-1", "Status": "COMPLETED"},
),
"packages": regional_store(
package_id,
{"PackageID": package_id, "PackageName": "synonyms"},
),
"domain_packages": regional_store(
association_key,
{"PackageID": package_id, "DomainName": domain_name},
),
}
persistence.save_state("opensearch", state)
first_snapshot = _json.loads((tmp_path / "opensearch.json").read_text())
assert first_snapshot["__ministack_format__"] == 3
loaded = persistence.load_state("opensearch")
assert loaded is not None
persistence.save_state("opensearch", loaded)
second_snapshot = _json.loads((tmp_path / "opensearch.json").read_text())
assert second_snapshot == first_snapshot
def test_opensearch_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
domains = AccountRegionScopedDict()
domains.set_scoped(
"123456789012",
"us-west-2",
"regional-domain",
{
"DomainName": "regional-domain",
"ARN": "arn:aws:es:us-west-2:123456789012:domain/regional-domain",
},
)
persistence.save_state("opensearch", {"domains": domains})
raw = _json.loads((tmp_path / "opensearch.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_domains = persistence.load_state("opensearch")["domains"]
assert loaded_domains.get_scoped(
"123456789012", "us-west-2", "regional-domain"
)["DomainName"] == "regional-domain"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("opensearch") is None
def test_apigateway_v2_legacy_state_uses_api_region_sidecar_and_tag_arns(
monkeypatch, tmp_path
):
import json as _json
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import apigateway as service
original_account = get_account_id()
original_region = get_region()
account_id = "000000000000"
boot_region = "us-east-1"
api_region = "us-west-2"
api_id = "legacy-http-api"
fallback_api_id = "legacy-fallback-api"
def scoped(value_by_api):
store = AccountScopedDict()
for key, value in value_by_api.items():
store.set_scoped(account_id, None, key, value)
return store
payload = {
"apis": scoped(
{
api_id: {"apiId": api_id, "name": "west-api"},
fallback_api_id: {
"apiId": fallback_api_id,
"name": "fallback-api",
},
}
),
"api_regions": scoped({api_id: api_region}),
"routes": scoped({api_id: {"route-1": {"routeId": "route-1"}}}),
"integrations": scoped(
{api_id: {"int-1": {"integrationId": "int-1"}}}
),
"stages": scoped({api_id: {"prod": {"stageName": "prod"}}}),
"deployments": scoped({api_id: {"dep-1": {"deploymentId": "dep-1"}}}),
"authorizers": scoped(
{api_id: {"auth-1": {"authorizerId": "auth-1"}}}
),
"route_responses": scoped(
{api_id: {"route-1": {"rr-1": {"routeResponseId": "rr-1"}}}}
),
"integration_responses": scoped(
{
api_id: {
"int-1": {
"ir-1": {"integrationResponseId": "ir-1"}
}
}
}
),
"api_tags": scoped(
{
f"arn:aws:apigateway:{api_region}::/apis/{api_id}": {
"env": "west"
}
}
),
}
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
(tmp_path / "apigateway.json").write_text(
_json.dumps(
{"__ministack_format__": 2, "payload": payload},
default=persistence._json_default,
)
)
loaded = persistence.load_state("apigateway")
service.reset()
try:
set_request_account_id(account_id)
set_request_region(boot_region)
service.load_persisted_state(loaded)
assert service._apis.get_scoped(account_id, api_region, api_id)[
"name"
] == "west-api"
assert service._routes.get_scoped(account_id, api_region, api_id) == {
"route-1": {"routeId": "route-1"}
}
assert service._integrations.get_scoped(
account_id, api_region, api_id
) == {"int-1": {"integrationId": "int-1"}}
assert service._stages.get_scoped(account_id, api_region, api_id) == {
"prod": {"stageName": "prod"}
}
assert service._deployments.get_scoped(
account_id, api_region, api_id
) == {"dep-1": {"deploymentId": "dep-1"}}
assert service._authorizers.get_scoped(
account_id, api_region, api_id
) == {"auth-1": {"authorizerId": "auth-1"}}
assert service._route_responses.get_scoped(
account_id, api_region, api_id
) == {"route-1": {"rr-1": {"routeResponseId": "rr-1"}}}
assert service._integration_responses.get_scoped(
account_id, api_region, api_id
) == {
"int-1": {"ir-1": {"integrationResponseId": "ir-1"}}
}
assert service._api_tags.get_scoped(
account_id,
api_region,
f"arn:aws:apigateway:{api_region}::/apis/{api_id}",
) == {"env": "west"}
assert (
service._apis.get_scoped(account_id, boot_region, fallback_api_id)[
"name"
]
== "fallback-api"
)
assert service._apis.get_scoped(account_id, boot_region, api_id) is None
finally:
service.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_apigateway_v2_region_scoped_state_round_trips_and_rejects_v2_reader(
monkeypatch, tmp_path
):
import json as _json
from ministack.core.responses import AccountRegionScopedDict
from ministack.services import apigateway as service
account_id = "000000000000"
api_id = "regional-http-api"
api_region = "us-west-2"
apis = AccountRegionScopedDict()
apis.set_scoped(
account_id,
api_region,
api_id,
{"apiId": api_id, "name": "regional API"},
)
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
persistence.save_state("apigateway", {"apis": apis})
raw = _json.loads((tmp_path / "apigateway.json").read_text())
assert raw["__ministack_format__"] == 3
loaded = persistence.load_state("apigateway")
service.reset()
try:
service.load_persisted_state(loaded)
assert service._apis.get_scoped(account_id, api_region, api_id)[
"name"
] == "regional API"
finally:
service.reset()
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("apigateway") is None
def test_apigateway_v1_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
"""A rollback binary must reject REST API regional state instead of
accepting it as v2 and silently dropping non-boot-region APIs."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
rest_apis = AccountRegionScopedDict()
rest_apis.set_scoped(
"123456789012",
"us-west-2",
"regionalapi",
{"id": "regionalapi", "name": "regional API"},
)
persistence.save_state("apigateway_v1", {"rest_apis": rest_apis})
raw = _json.loads((tmp_path / "apigateway_v1.json").read_text())
assert raw["__ministack_format__"] == 3
loaded = persistence.load_state("apigateway_v1")["rest_apis"]
assert loaded.get_scoped("123456789012", "us-west-2", "regionalapi")[
"name"
] == "regional API"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("apigateway_v1") is None
def test_opensearch_is_registered_for_persistence_save():
from ministack.app import _build_persistence_save_dict, _state_map
from ministack.services import opensearch
assert _state_map["opensearch"] == "opensearch"
opensearch.reset()
try:
opensearch._domains["persisted-domain"] = {
"DomainName": "persisted-domain",
"ARN": "arn:aws:es:us-east-1:000000000000:domain/persisted-domain",
}
save_dict = _build_persistence_save_dict()
assert "opensearch" in save_dict
assert save_dict["opensearch"]()["domains"]["persisted-domain"][
"DomainName"
] == "persisted-domain"
finally:
opensearch.reset()
def test_opensearch_legacy_state_migrates_domain_children_to_parent_region():
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import opensearch
original_account = get_account_id()
original_region = get_region()
account_id = "123456789012"
other_account = "210987654321"
boot_region = "us-east-1"
domain_region = "eu-west-1"
other_region = "us-west-2"
domain_name = "legacy-domain"
peer_domain_name = "peer-domain"
other_domain_name = "other-domain"
package_id = "FPKGDOMAIN1"
orphan_package_id = "FPKGBOOT001"
domain_package_key = f"{package_id}:{domain_name}"
peer_domain_package_key = f"{package_id}:{peer_domain_name}"
orphan_key = "FPKGORPHAN:missing-domain"
other_key = f"FPKGOTHER:{other_domain_name}"
def legacy_store(*entries):
store = AccountScopedDict()
for account, key, value in entries:
store._data[(account, key)] = value
return store
try:
set_request_account_id(account_id)
set_request_region(boot_region)
opensearch.reset()
opensearch.load_persisted_state({
"domains": legacy_store(
(
account_id,
peer_domain_name,
{
"DomainName": peer_domain_name,
"ARN": f"arn:aws:es:{other_region}:{account_id}:domain/{peer_domain_name}",
},
),
(
account_id,
domain_name,
{
"DomainName": domain_name,
"ARN": f"arn:aws:es:{domain_region}:{account_id}:domain/{domain_name}",
},
),
(
other_account,
other_domain_name,
{
"DomainName": other_domain_name,
"ARN": f"arn:aws:es:{other_region}:{other_account}:domain/{other_domain_name}",
},
),
),
"change_progress": legacy_store(
(account_id, domain_name, {"ChangeId": "change-1"}),
(other_account, other_domain_name, {"ChangeId": "change-2"}),
(account_id, "missing-domain", {"ChangeId": "orphan-change"}),
),
"packages": legacy_store(
(
account_id,
package_id,
{
"PackageID": package_id,
"PackageName": "associated-package",
"PackageOptions": {"analysis": {"enabled": True}},
},
),
(
account_id,
orphan_package_id,
{
"PackageID": orphan_package_id,
"PackageName": "boot-package",
"PackageDescription": (
"user text arn:aws:lambda:us-west-2:123456789012:function:nope"
),
},
),
),
"domain_packages": legacy_store(
(
account_id,
domain_package_key,
{"PackageID": package_id, "DomainName": domain_name},
),
(
account_id,
peer_domain_package_key,
{"PackageID": package_id, "DomainName": peer_domain_name},
),
(
account_id,
orphan_key,
{"PackageID": "FPKGORPHAN", "DomainName": "missing-domain"},
),
(
other_account,
other_key,
{"PackageID": "FPKGOTHER", "DomainName": other_domain_name},
),
),
})
assert opensearch._domains.get_scoped(
account_id, domain_region, domain_name
)["DomainName"] == domain_name
assert opensearch._change_progress.get_scoped(
account_id, domain_region, domain_name
)["ChangeId"] == "change-1"
assert opensearch._domain_packages.get_scoped(
account_id, domain_region, domain_package_key
)["DomainName"] == domain_name
assert opensearch._domain_packages.get_scoped(
account_id, boot_region, domain_package_key
) is None
assert opensearch._packages.get_scoped(
account_id, domain_region, package_id
)["PackageName"] == "associated-package"
peer_package = opensearch._packages.get_scoped(
account_id, other_region, package_id
)
assert peer_package["PackageName"] == "associated-package"
assert opensearch._domain_packages.get_scoped(
account_id, other_region, peer_domain_package_key
)["DomainName"] == peer_domain_name
package = opensearch._packages.get_scoped(
account_id, domain_region, package_id
)
assert package is not peer_package
package["PackageOptions"]["analysis"]["enabled"] = False
assert peer_package["PackageOptions"]["analysis"]["enabled"] is True
assert opensearch._packages.get_scoped(
account_id, boot_region, package_id
) is None
assert opensearch._packages.get_scoped(
account_id, boot_region, orphan_package_id
)["PackageName"] == "boot-package"
assert opensearch._change_progress.get_scoped(
account_id, boot_region, "missing-domain"
)["ChangeId"] == "orphan-change"
assert opensearch._domain_packages.get_scoped(
account_id, boot_region, orphan_key
)["DomainName"] == "missing-domain"
assert opensearch._change_progress.get_scoped(
other_account, other_region, other_domain_name
)["ChangeId"] == "change-2"
assert opensearch._domain_packages.get_scoped(
other_account, other_region, other_key
)["DomainName"] == other_domain_name
finally:
opensearch.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_opensearch_load_recreates_dataplane_in_domain_region(monkeypatch):
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import opensearch
original_account = get_account_id()
original_region = get_region()
account_id = "123456789012"
boot_region = "us-east-1"
domain_region = "us-west-2"
domain_name = "restored-domain"
calls = []
def legacy_store(*entries):
store = AccountScopedDict()
for account, key, value in entries:
store._data[(account, key)] = value
return store
def fake_spawn(domain, engine_version):
calls.append((get_account_id(), get_region(), domain, engine_version))
return (
f"{domain}.{get_region()}.ministack.local",
9200,
f"new-{get_region()}",
f"{domain}.{get_region()}.dashboards.local:5601",
f"dash-{get_region()}",
)
def fake_teardown_dataplane(rec):
calls.append(
(
"teardown_ids",
get_account_id(),
get_region(),
rec.get("_ContainerId"),
rec.get("_DashboardsContainerId"),
)
)
def fake_teardown_named_dataplane(domain):
calls.append(("teardown_name", get_account_id(), get_region(), domain))
monkeypatch.setattr(opensearch, "_spawn_dataplane", fake_spawn)
monkeypatch.setattr(opensearch, "_teardown_dataplane", fake_teardown_dataplane)
monkeypatch.setattr(
opensearch, "_teardown_named_dataplane", fake_teardown_named_dataplane
)
try:
set_request_account_id(account_id)
set_request_region(boot_region)
opensearch.reset()
opensearch.load_persisted_state({
"domains": legacy_store(
(
account_id,
domain_name,
{
"DomainName": domain_name,
"ARN": f"arn:aws:es:{domain_region}:{account_id}:domain/{domain_name}",
"EngineVersion": "OpenSearch_2.11",
"Endpoint": "stale.local:9200",
"_Endpoint": "stale.local:9200",
"_ContainerId": "stale-container",
"_DashboardsContainerId": "stale-dashboard",
"DashboardEndpoint": "stale-dashboard.local:5601",
},
),
),
})
assert calls == [
(
"teardown_ids",
account_id,
domain_region,
"stale-container",
"stale-dashboard",
),
("teardown_name", account_id, domain_region, domain_name),
(account_id, domain_region, domain_name, "OpenSearch_2.11"),
]
assert get_account_id() == account_id
assert get_region() == boot_region
restored = opensearch._domains.get_scoped(
account_id, domain_region, domain_name
)
assert restored["Endpoint"] == (
f"{domain_name}.{domain_region}.ministack.local:9200"
)
assert restored["_Endpoint"] == restored["Endpoint"]
assert restored["_ContainerId"] == f"new-{domain_region}"
assert restored["_DashboardsContainerId"] == f"dash-{domain_region}"
assert restored["DashboardEndpoint"] == (
f"{domain_name}.{domain_region}.dashboards.local:5601"
)
finally:
opensearch.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_servicediscovery_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
"""A rollback binary must reject Cloud Map's regional schema instead of
accepting it as v2 and silently dropping every regional store."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
namespaces = AccountRegionScopedDict()
namespaces.set_scoped(
"000000000000",
"us-west-2",
"ns-regional",
{
"Arn": (
"arn:aws:servicediscovery:us-west-2:000000000000:"
"namespace/ns-regional"
)
},
)
persistence.save_state("servicediscovery", {"namespaces": namespaces})
raw = _json.loads((tmp_path / "servicediscovery.json").read_text())
assert raw["__ministack_format__"] == 3
loaded_namespaces = persistence.load_state("servicediscovery")["namespaces"]
assert loaded_namespaces.get_scoped(
"000000000000", "us-west-2", "ns-regional"
)["Arn"].endswith("namespace/ns-regional")
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("servicediscovery") is None
def test_glue_region_scoped_state_round_trips_and_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):
"""Glue's same-key regional state must round-trip, while a v2 rollback
refuses the regional schema instead of silently collapsing partitions."""
import json as _json
from ministack.core.responses import AccountRegionScopedDict
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
account = "000000000000"
databases = AccountRegionScopedDict()
job_runs = AccountRegionScopedDict()
databases.set_scoped(
account,
"us-east-1",
"same-name",
{"Name": "same-name", "Description": "east"},
)
databases.set_scoped(
account,
"us-west-2",
"same-name",
{"Name": "same-name", "Description": "west"},
)
job_runs.set_scoped(
account,
"us-east-1",
"same-job",
[{"Id": "east-run", "JobName": "same-job"}],
)
job_runs.set_scoped(
account,
"us-west-2",
"same-job",
[{"Id": "west-run", "JobName": "same-job"}],
)
persistence.save_state(
"glue", {"databases": databases, "job_runs": job_runs}
)
raw = _json.loads((tmp_path / "glue.json").read_text())
assert raw["__ministack_format__"] == 3
loaded = persistence.load_state("glue")
assert loaded["databases"].get_scoped(
account, "us-east-1", "same-name"
)["Description"] == "east"
assert loaded["databases"].get_scoped(
account, "us-west-2", "same-name"
)["Description"] == "west"
assert loaded["job_runs"].get_scoped(
account, "us-east-1", "same-job"
)[0]["Id"] == "east-run"
assert loaded["job_runs"].get_scoped(
account, "us-west-2", "same-job"
)[0]["Id"] == "west-run"
monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
assert persistence.load_state("glue") is None
def test_batch_persistence_lifecycle_restores_regional_state(monkeypatch, tmp_path):
"""The registry loader preserves state outside the ambient boot region."""
from ministack.app import _build_persistence_save_dict, _state_map
from ministack.core.responses import set_request_account_id, set_request_region
from ministack.services import batch as service
account_id = "111111111111"
boot_region = "us-east-1"
resource_region = "us-west-2"
job_id = "regional-job"
job = {
"jobArn": f"arn:aws:batch:{resource_region}:{account_id}:job/{job_id}",
"status": "SUCCEEDED",
}
monkeypatch.setattr(persistence, "PERSIST_STATE", True)
monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
set_request_account_id(account_id)
set_request_region(boot_region)
service.reset()
try:
service._jobs.set_scoped(account_id, resource_region, job_id, job)
assert _state_map["batch"] == "batch"
save_dict = _build_persistence_save_dict()
assert "batch" in save_dict
persistence.save_all({"batch": save_dict["batch"]})
service.reset()
service.load_persisted_state(persistence.load_state("batch"))
assert service._jobs.get_scoped(
account_id, resource_region, job_id
) == job
assert service._jobs.get_scoped(account_id, boot_region, job_id) is None
finally:
service.reset()
# ── signer._jobs / signer._profiles / signer._tokens ───────────────────
def test_signer_jobs_profiles_and_tokens_survive_warm_boot():
"""A restored signer job must keep its signedObject reference: a caller
whose contract is the signed object at `prefix + jobId` reads that key,
and a job record that forgets where its marker lives can no longer
answer DescribeSigningJob for it after a restart. The idempotency map
must survive too, or a retried Start after a restart would sign twice."""
mod = _get_module("signer")
mod.reset()
job_id = "persisted-signing-job"
now = 1700000000
try:
mod._profiles["persisted_profile"] = {
"profileName": "persisted_profile",
"profileVersion": "abc123def4",
"profileVersionArn": (
"arn:aws:signer:us-east-1:000000000000:"
"/signing-profiles/persisted_profile/abc123def4"
),
"arn": (
"arn:aws:signer:us-east-1:000000000000:"
"/signing-profiles/persisted_profile"
),
"platformId": "AWSIoTDeviceManagement-SHA256-ECDSA",
"status": "Active",
}
mod._jobs[job_id] = {
"jobId": job_id,
"source": {"s3": {"bucketName": "src-bkt", "key": "fw.bin",
"version": "null"}},
"signedObject": {"s3": {"bucketName": "dst-bkt",
"key": f"signed/{job_id}"}},
"profileName": "persisted_profile",
"profileVersion": "abc123def4",
"platformId": "AWSIoTDeviceManagement-SHA256-ECDSA",
"status": "Succeeded",
"createdAt": now,
"completedAt": now,
"requestedBy": "arn:aws:iam::000000000000:root",
"jobOwner": "000000000000",
"jobInvoker": "000000000000",
}
mod._tokens["token-1"] = {"jobId": job_id, "jobOwner": "000000000000"}
_round_trip_dict(mod, "signer")
job = mod._jobs.get(job_id)
assert job is not None, "_jobs lost across save_state -> load_state"
assert job["status"] == "Succeeded"
assert job["signedObject"]["s3"]["key"] == f"signed/{job_id}"
profile = mod._profiles.get("persisted_profile")
assert profile is not None, (
"_profiles lost across save_state -> load_state"
)
assert profile["profileVersion"] == "abc123def4"
assert mod._tokens.get("token-1") == {
"jobId": job_id, "jobOwner": "000000000000",
}
finally:
mod.reset()
def _ec2_va_populate(mod, account_id, region, label):
"""Create one of each Verified Access resource through the handlers, in one scope."""
import json
from ministack.core.responses import request_scope
from ministack.services import acm
oidc = {"Issuer": "https://idp.example.com", "AuthorizationEndpoint": "https://idp.example.com/a",
"TokenEndpoint": "https://idp.example.com/t", "UserInfoEndpoint": "https://idp.example.com/u",
"ClientId": "c", "ClientSecret": "s", "Scope": "openid"}
with request_scope(account_id, region):
mod._ensure_defaults_initialized()
status, _, body = mod._create_verified_access_trust_provider({
"TrustProviderType": "user", "UserTrustProviderType": "oidc", "PolicyReferenceName": label,
**{f"OidcOptions.{k}": v for k, v in oidc.items()},
"TagSpecification.1.ResourceType": "verified-access-trust-provider",
"TagSpecification.1.Tag.1.Key": "owner", "TagSpecification.1.Tag.1.Value": label,
})
assert status == 200, body
tp_id = next(iter(mod._va_trust_providers))
status, _, body = mod._create_verified_access_instance({
"Description": label,
"TagSpecification.1.ResourceType": "verified-access-instance",
"TagSpecification.1.Tag.1.Key": "owner", "TagSpecification.1.Tag.1.Value": label,
})
assert status == 200, body
inst_id = next(iter(mod._va_instances))
assert mod._attach_verified_access_trust_provider({
"VerifiedAccessInstanceId": inst_id, "VerifiedAccessTrustProviderId": tp_id})[0] == 200
assert mod._modify_verified_access_instance_logging_configuration({
"VerifiedAccessInstanceId": inst_id, "AccessLogs.IncludeTrustContext": "true"})[0] == 200
assert mod._create_verified_access_group({
"VerifiedAccessInstanceId": inst_id, "PolicyDocument": f"permit-{label}"})[0] == 200
group_id = next(iter(mod._va_groups))
enis = set(mod._network_interfaces)
assert mod._create_network_interface({"SubnetId": mod._DEFAULT_SUBNET_ID})[0] == 200
eni_id = next(iter(set(mod._network_interfaces) - enis))
cert_arn = json.loads(acm._request_certificate({"DomainName": f"{label}.example.com"})[2])["CertificateArn"]
groups = set(mod._security_groups)
assert mod._create_security_group({"GroupName": f"va-{label}", "GroupDescription": "va",
"VpcId": mod._DEFAULT_VPC_ID})[0] == 200
sg_id = next(iter(set(mod._security_groups) - groups))
status, _, body = mod._create_verified_access_endpoint({
"VerifiedAccessGroupId": group_id, "EndpointType": "network-interface", "AttachmentType": "vpc",
"DomainCertificateArn": cert_arn, "ApplicationDomain": f"{label}.example.com", "EndpointDomainPrefix": label,
"NetworkInterfaceOptions.NetworkInterfaceId": eni_id, "NetworkInterfaceOptions.Port": "443",
"NetworkInterfaceOptions.Protocol": "https", "SecurityGroupId.1": sg_id,
})
assert status == 200, body
ep_id = next(iter(mod._va_endpoints))
return {"tp": tp_id, "inst": inst_id, "group": group_id, "ep": ep_id}
def test_ec2_verified_access_round_trip_across_accounts():
from ministack.core.responses import request_scope
scopes = (("111111111111", "us-east-1", "a"), ("222222222222", "eu-west-1", "b"))
ids = {}
def populate(mod):
for account_id, region, label in scopes:
ids[label] = _ec2_va_populate(mod, account_id, region, label)
def observe(mod):
for account_id, region, label in scopes:
got = ids[label]
with request_scope(account_id, region):
tp = mod._va_trust_providers.get(got["tp"])
assert tp["PolicyReferenceName"] == label
assert tp["OidcOptions"]["ClientSecret"] == "s"
inst = mod._va_instances.get(got["inst"])
assert inst["Description"] == label
assert inst["TrustProviderIds"] == [got["tp"]]
assert inst["AccessLogs"]["IncludeTrustContext"] is True
assert mod._va_groups.get(got["group"])["PolicyDocument"] == f"permit-{label}"
assert mod._va_endpoints.get(got["ep"])["VerifiedAccessGroupId"] == got["group"]
for resource_id in (got["tp"], got["inst"]):
assert mod._tags.get(resource_id) == [{"Key": "owner", "Value": label}]
# The other tenant's resources stay invisible from this scope.
other = ids["b" if label == "a" else "a"]
with request_scope(account_id, region):
assert other["inst"] not in mod._va_instances
_round_trip("ec2", "ec2", populate, observe)
def test_ec2_verified_access_reset_clears_stores():
from ministack.core.responses import request_scope
mod = _module("ec2")
mod.reset()
got = _ec2_va_populate(mod, "111111111111", "us-east-1", "r")
mod.reset()
with request_scope("111111111111", "us-east-1"):
assert got["inst"] not in mod._va_instances
assert got["tp"] not in mod._va_trust_providers
assert got["group"] not in mod._va_groups
assert got["ep"] not in mod._va_endpoints
assert mod._tags.get(got["inst"]) is None