Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
feat(athena): scope state by region (#1168)
Ian Oberst86c685dda4ad
diff --git a/ministack/core/persistence.py b/ministack/core/persistence.pyindex 5d0f02f0..cbc26722 100644--- a/ministack/core/persistence.py+++ b/ministack/core/persistence.py@@ -27,6 +27,7 @@ STATE_DIR = os.environ.get("STATE_DIR", "/tmp/ministack-state")# still load and migrate. (U4)STATE_FORMAT_VERSION = 2SERVICE_STATE_FORMAT_VERSIONS = {+ "athena": 3,"batch": 3,"ecs": 3,"resource_groups": 3,diff --git a/ministack/services/athena.py b/ministack/services/athena.pyindex 022f7cbc..29f8cc90 100644--- a/ministack/services/athena.py+++ b/ministack/services/athena.py@@ -29,12 +29,14 @@ from urllib.parse import urlparsefrom ministack.core.arn import ArnParseError, parse_arnfrom ministack.core.persistence import PERSIST_STATE, load_statefrom ministack.core.responses import (+ AccountRegionScopedDict,AccountScopedDict,error_response_json,get_account_id,get_region,json_response,new_uuid,+ set_request_region,)logger = logging.getLogger("athena")@@ -55,14 +57,13 @@ def get_athena_engine():return engine-_executions = AccountScopedDict()-# Per-account workgroups / data catalogs. AWS's "primary" workgroup and-# "AwsDataCatalog" exist in every account — we lazily seed them per-tenant-# on first access so two accounts never share the same workgroup or catalog-# state (creation times, configs, etc.).-_workgroups = AccountScopedDict()-_named_queries = AccountScopedDict()-_data_catalogs = AccountScopedDict()+_executions = AccountRegionScopedDict()+# Per-account-and-region workgroups / data catalogs. AWS's "primary" workgroup+# and "AwsDataCatalog" exist in every region — we lazily seed them per scope on+# first access so requests never share workgroup or catalog state.+_workgroups = AccountRegionScopedDict()+_named_queries = AccountRegionScopedDict()+_data_catalogs = AccountRegionScopedDict()def _ensure_default_workgroup():@@ -86,7 +87,7 @@ def _ensure_default_data_catalog():"Type": "GLUE","Parameters": {},}-_prepared_statements = AccountScopedDict() # "workgroup/name" -> statement dict+_prepared_statements = AccountRegionScopedDict() # "workgroup/name" -> statement dict_tags = AccountScopedDict() # arn -> {key: value, ...}@@ -104,21 +105,94 @@ def get_state():def restore_state(data):- # AccountScopedDicts are mutated in-place — no module-level reassignment.+ # Scoped dicts are mutated in-place — no module-level reassignment._executions.clear()- _executions.update(data.get("_executions", {}))_workgroups.clear()- _workgroups.update(data.get("_workgroups", {}))_named_queries.clear()- _named_queries.update(data.get("_named_queries", {}))_data_catalogs.clear()- _data_catalogs.update(data.get("_data_catalogs", {}))_prepared_statements.clear()- _prepared_statements.update(data.get("_prepared_statements", {}))_tags.clear()++ legacy_workgroup_regions = _restore_regional_store(+ _workgroups, data.get("_workgroups", {})+ )+ for store, key, workgroup_fields in (+ (_executions, "_executions", ("WorkGroup",)),+ (_named_queries, "_named_queries", ("WorkGroup",)),+ (+ _prepared_statements,+ "_prepared_statements",+ ("WorkGroupName", "WorkGroup"),+ ),+ ):+ _restore_workgroup_child_store(+ store,+ data.get(key, {}),+ legacy_workgroup_regions,+ workgroup_fields,+ )+ _restore_regional_store(_data_catalogs, data.get("_data_catalogs", {}))_tags.update(data.get("_tags", {}))+def _legacy_region_for_value(store, key, value):+ """Infer an ARN region, falling back to Athena's configured boot region."""+ request_region = get_region()+ try:+ set_request_region(REGION)+ return store._region_for_legacy_value(key, value)+ finally:+ set_request_region(request_region)+++def _restore_regional_store(store, restored):+ """Restore current state verbatim and legacy state deterministically."""+ if isinstance(restored, AccountRegionScopedDict):+ store.update(restored)+ return {}++ restored_regions = {}+ if isinstance(restored, AccountScopedDict):+ items = restored._data.items()+ else:+ account_id = get_account_id()+ items = (((account_id, key), value) for key, value in restored.items())++ for (account_id, key), value in items:+ region = _legacy_region_for_value(store, key, value)+ store.set_scoped(account_id, region, key, value)+ restored_regions[(account_id, key)] = region+ return restored_regions+++def _restore_workgroup_child_store(+ store, restored, workgroup_regions, workgroup_fields+):+ """Place legacy child records beside their referenced workgroup."""+ if isinstance(restored, AccountRegionScopedDict):+ store.update(restored)+ return++ if isinstance(restored, AccountScopedDict):+ items = restored._data.items()+ else:+ account_id = get_account_id()+ items = (((account_id, key), value) for key, value in restored.items())++ for (account_id, key), value in items:+ workgroup = next(+ (value.get(field) for field in workgroup_fields if value.get(field)),+ None,+ )+ if not workgroup and store is _prepared_statements:+ workgroup = str(key).partition("/")[0]+ workgroup = workgroup or "primary"+ region = workgroup_regions.get((account_id, workgroup))+ if region is None:+ region = _legacy_region_for_value(store, key, value)+ store.set_scoped(account_id, region, key, value)++try:_restored = load_state("athena")if _restored:diff --git a/tests/test_athena.py b/tests/test_athena.pyindex b102e3ca..606287d8 100644--- a/tests/test_athena.py+++ b/tests/test_athena.py@@ -6,10 +6,23 @@ import uuid as _uuid_modimport zipfilefrom urllib.parse import urlparse+import boto3import pytest+from botocore.config import Configfrom botocore.exceptions import ClientError+def _client(region):+ return boto3.client(+ "athena",+ endpoint_url=os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566"),+ region_name=region,+ aws_access_key_id="test",+ aws_secret_access_key="test",+ config=Config(retries={"mode": "standard"}),+ )++def test_athena_query(athena):resp = athena.start_query_execution(QueryString="SELECT 1 AS num, 'hello' AS greeting",@@ -91,6 +104,273 @@ def test_athena_workgroup_v2(athena):with pytest.raises(ClientError):athena.get_work_group(WorkGroup="ath-wg-v2")++def test_athena_named_resources_are_region_scoped():+ east = _client("us-east-1")+ west = _client("us-west-2")+ suffix = _uuid_mod.uuid4().hex[:8]+ workgroup = f"ath-region-wg-{suffix}"+ catalog = f"ath-region-catalog-{suffix}"+ statement = f"ath-region-statement-{suffix}"++ for client, region in ((east, "east"), (west, "west")):+ client.create_work_group(Name=workgroup, Description=region)+ client.create_data_catalog(Name=catalog, Type="HIVE", Description=region)+ client.create_prepared_statement(+ StatementName=statement,+ WorkGroup=workgroup,+ QueryStatement=f"SELECT '{region}'",+ )++ try:+ assert east.get_work_group(WorkGroup=workgroup)["WorkGroup"][+ "Description"+ ] == "east"+ assert west.get_work_group(WorkGroup=workgroup)["WorkGroup"][+ "Description"+ ] == "west"+ assert east.get_data_catalog(Name=catalog)["DataCatalog"][+ "Description"+ ] == "east"+ assert west.get_data_catalog(Name=catalog)["DataCatalog"][+ "Description"+ ] == "west"+ assert east.get_prepared_statement(+ StatementName=statement, WorkGroup=workgroup+ )["PreparedStatement"]["QueryStatement"] == "SELECT 'east'"+ assert west.get_prepared_statement(+ StatementName=statement, WorkGroup=workgroup+ )["PreparedStatement"]["QueryStatement"] == "SELECT 'west'"+ finally:+ for client in (east, west):+ client.delete_prepared_statement(+ StatementName=statement, WorkGroup=workgroup+ )+ client.delete_data_catalog(Name=catalog)+ client.delete_work_group(+ WorkGroup=workgroup, RecursiveDeleteOption=True+ )+++def test_athena_defaults_are_seeded_per_region():+ from ministack.core.responses import (+ get_account_id,+ get_region,+ set_request_account_id,+ set_request_region,+ )+ from ministack.services import athena as service++ original_account = get_account_id()+ original_region = get_region()+ account_id = "111111111111"++ service.reset()+ try:+ set_request_account_id(account_id)+ set_request_region("us-east-1")+ service._ensure_default_workgroup()+ service._ensure_default_data_catalog()+ service._workgroups["primary"]["Description"] = "east primary"++ set_request_region("us-west-2")+ service._ensure_default_workgroup()+ service._ensure_default_data_catalog()++ assert service._workgroups["primary"]["Description"] == "Primary workgroup"+ assert service._data_catalogs["AwsDataCatalog"]["Type"] == "GLUE"+ assert service._workgroups.get_scoped(+ account_id, "us-east-1", "primary"+ )["Description"] == "east primary"+ assert service._data_catalogs.get_scoped(+ account_id, "us-east-1", "AwsDataCatalog"+ )["Type"] == "GLUE"+ finally:+ service.reset()+ set_request_account_id(original_account)+ set_request_region(original_region)+++def test_athena_legacy_state_migrates_to_configured_boot_region(monkeypatch):+ from ministack.core.responses import (+ AccountScopedDict,+ get_account_id,+ get_region,+ set_request_account_id,+ set_request_region,+ )+ from ministack.services import athena as service++ original_account = get_account_id()+ original_region = get_region()+ account_id = "111111111111"+ boot_region = "us-west-2"+ first_request_region = "eu-central-1"+ tags = AccountScopedDict()+ stores = {+ "_executions": ("legacy-execution", {"QueryExecutionId": "legacy-execution"}),+ "_workgroups": ("legacy-workgroup", {"Name": "legacy-workgroup"}),+ "_named_queries": ("legacy-query", {"NamedQueryId": "legacy-query"}),+ "_data_catalogs": ("legacy-catalog", {"Name": "legacy-catalog"}),+ "_prepared_statements": (+ "legacy-workgroup/legacy-statement",+ {"StatementName": "legacy-statement", "WorkGroup": "legacy-workgroup"},+ ),+ }++ set_request_account_id(account_id)+ monkeypatch.setattr(service, "REGION", boot_region)+ set_request_region(first_request_region)+ payload = {}+ for store_name, (key, value) in stores.items():+ legacy = AccountScopedDict()+ legacy[key] = value+ payload[store_name] = legacy+ tag_arn = f"arn:aws:athena:{boot_region}:{account_id}:workgroup/legacy-workgroup"+ tags[tag_arn] = {"legacy": "true"}+ payload["_tags"] = tags++ service.reset()+ try:+ service.restore_state(payload)+ for store_name, (key, value) in stores.items():+ restored = getattr(service, store_name).get_scoped(+ account_id, boot_region, key+ )+ assert restored == value+ assert (+ getattr(service, store_name).get_scoped(+ account_id, first_request_region, key+ )+ is None+ )+ assert service._tags.get(tag_arn) == {"legacy": "true"}+ assert get_region() == first_request_region+ finally:+ service.reset()+ set_request_account_id(original_account)+ set_request_region(original_region)+++def test_athena_legacy_children_follow_workgroup_region(monkeypatch):+ from ministack.core.responses import (+ AccountScopedDict,+ get_account_id,+ get_region,+ set_request_account_id,+ set_request_region,+ )+ from ministack.services import athena as service++ original_account = get_account_id()+ original_region = get_region()+ account_id = "111111111111"+ boot_region = "us-east-1"+ workgroup_region = "us-west-2"+ first_request_region = "eu-central-1"+ workgroup_name = "legacy-workgroup"++ def legacy_store(key, value):+ store = AccountScopedDict()+ store._data[(account_id, key)] = value+ return store++ payload = {+ "_workgroups": legacy_store(+ workgroup_name,+ {+ "Name": workgroup_name,+ "Configuration": {+ "ResultConfiguration": {+ "EncryptionConfiguration": {+ "KmsKey": (+ f"arn:aws:kms:{workgroup_region}:{account_id}:key/key-id"+ )+ }+ }+ },+ },+ ),+ "_executions": legacy_store(+ "legacy-execution",+ {"QueryExecutionId": "legacy-execution", "WorkGroup": workgroup_name},+ ),+ "_named_queries": legacy_store(+ "legacy-query",+ {"NamedQueryId": "legacy-query", "WorkGroup": workgroup_name},+ ),+ "_prepared_statements": legacy_store(+ f"{workgroup_name}/legacy-statement",+ {+ "StatementName": "legacy-statement",+ "WorkGroupName": workgroup_name,+ },+ ),+ "_data_catalogs": legacy_store(+ "legacy-catalog", {"Name": "legacy-catalog"}+ ),+ }++ monkeypatch.setattr(service, "REGION", boot_region)+ set_request_account_id(account_id)+ set_request_region(first_request_region)+ service.reset()+ try:+ assert service.REGION != first_request_region+ service.restore_state(payload)++ assert service._workgroups.get_scoped(+ account_id, workgroup_region, workgroup_name+ )+ for store_name, key in (+ ("_executions", "legacy-execution"),+ ("_named_queries", "legacy-query"),+ ("_prepared_statements", f"{workgroup_name}/legacy-statement"),+ ):+ store = getattr(service, store_name)+ assert store.get_scoped(account_id, workgroup_region, key)+ assert store.get_scoped(account_id, boot_region, key) is None+ assert store.get_scoped(account_id, first_request_region, key) is None+ assert service._data_catalogs.get_scoped(+ account_id, boot_region, "legacy-catalog"+ )+ assert get_region() == first_request_region+ finally:+ service.reset()+ set_request_account_id(original_account)+ set_request_region(original_region)+++def test_reset_clears_athena_state_across_regions():+ from ministack.core.responses import get_region, set_request_region+ from ministack.services import athena as service++ original_region = get_region()+ regional_stores = (+ service._executions,+ service._workgroups,+ service._named_queries,+ service._data_catalogs,+ service._prepared_statements,+ )++ service.reset()+ try:+ for region in ("us-east-1", "us-west-2"):+ set_request_region(region)+ for store in regional_stores:+ store[f"resource-{region}"] = {"region": region}+ service._tags["arn:aws:athena:us-east-1:000000000000:workgroup/tagged"] = {+ "tag": "value"+ }++ service.reset()+ assert all(not store.has_any() for store in regional_stores)+ assert not service._tags._data+ finally:+ service.reset()+ set_request_region(original_region)+def test_athena_named_query_v2(athena):resp = athena.create_named_query(Name="ath-nq-v2",diff --git a/tests/test_persistence.py b/tests/test_persistence.pyindex 2d8a2eca..3de3fde3 100644--- a/tests/test_persistence.py+++ b/tests/test_persistence.py@@ -1832,6 +1832,39 @@ def test_ecs_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path)assert persistence.load_state("ecs") is None+def test_athena_region_scoped_state_is_rejected_by_v2_reader(+ monkeypatch, tmp_path+):+ """A rollback binary must reject Athena's regional schema instead of+ accepting it as v2 and silently dropping every regional store."""+ import json as _json++ from ministack.core.responses import AccountRegionScopedDict++ monkeypatch.setattr(persistence, "PERSIST_STATE", True)+ monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))++ workgroups = AccountRegionScopedDict()+ workgroups.set_scoped(+ "000000000000",+ "us-west-2",+ "regional-workgroup",+ {"Name": "regional-workgroup", "Description": "west"},+ )+ persistence.save_state("athena", {"_workgroups": workgroups})++ raw = _json.loads((tmp_path / "athena.json").read_text())+ assert raw["__ministack_format__"] == 3+ loaded_workgroups = persistence.load_state("athena")["_workgroups"]+ assert loaded_workgroups.get_scoped(+ "000000000000", "us-west-2", "regional-workgroup"+ )["Description"] == "west"++ # Simulate the previous binary, whose highest understood format is v2.+ monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})+ assert persistence.load_state("athena") is None++def test_resource_groups_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path):