Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

feat(athena): scope state by region (#1168)

Ian Oberst86c685dda4ad
diff --git a/ministack/core/persistence.py b/ministack/core/persistence.py
index 5d0f02f0..cbc26722 100644
--- a/ministack/core/persistence.py
+++ b/ministack/core/persistence.py
@@ -27,6 +27,7 @@ STATE_DIR = os.environ.get("STATE_DIR", "/tmp/ministack-state")
# still load and migrate. (U4)
STATE_FORMAT_VERSION = 2
SERVICE_STATE_FORMAT_VERSIONS = {
+ "athena": 3,
"batch": 3,
"ecs": 3,
"resource_groups": 3,
diff --git a/ministack/services/athena.py b/ministack/services/athena.py
index 022f7cbc..29f8cc90 100644
--- a/ministack/services/athena.py
+++ b/ministack/services/athena.py
@@ -29,12 +29,14 @@ from urllib.parse import urlparse
from ministack.core.arn import ArnParseError, parse_arn
from ministack.core.persistence import PERSIST_STATE, load_state
from ministack.core.responses import (
+ AccountRegionScopedDict,
AccountScopedDict,
error_response_json,
get_account_id,
get_region,
json_response,
new_uuid,
+ set_request_region,
)
logger = logging.getLogger("athena")
@@ -55,14 +57,13 @@ def get_athena_engine():
return engine
-_executions = AccountScopedDict()
-# Per-account workgroups / data catalogs. AWS's "primary" workgroup and
-# "AwsDataCatalog" exist in every account — we lazily seed them per-tenant
-# on first access so two accounts never share the same workgroup or catalog
-# state (creation times, configs, etc.).
-_workgroups = AccountScopedDict()
-_named_queries = AccountScopedDict()
-_data_catalogs = AccountScopedDict()
+_executions = AccountRegionScopedDict()
+# Per-account-and-region workgroups / data catalogs. AWS's "primary" workgroup
+# and "AwsDataCatalog" exist in every region — we lazily seed them per scope on
+# first access so requests never share workgroup or catalog state.
+_workgroups = AccountRegionScopedDict()
+_named_queries = AccountRegionScopedDict()
+_data_catalogs = AccountRegionScopedDict()
def _ensure_default_workgroup():
@@ -86,7 +87,7 @@ def _ensure_default_data_catalog():
"Type": "GLUE",
"Parameters": {},
}
-_prepared_statements = AccountScopedDict() # "workgroup/name" -> statement dict
+_prepared_statements = AccountRegionScopedDict() # "workgroup/name" -> statement dict
_tags = AccountScopedDict() # arn -> {key: value, ...}
@@ -104,21 +105,94 @@ def get_state():
def restore_state(data):
- # AccountScopedDicts are mutated in-place — no module-level reassignment.
+ # Scoped dicts are mutated in-place — no module-level reassignment.
_executions.clear()
- _executions.update(data.get("_executions", {}))
_workgroups.clear()
- _workgroups.update(data.get("_workgroups", {}))
_named_queries.clear()
- _named_queries.update(data.get("_named_queries", {}))
_data_catalogs.clear()
- _data_catalogs.update(data.get("_data_catalogs", {}))
_prepared_statements.clear()
- _prepared_statements.update(data.get("_prepared_statements", {}))
_tags.clear()
+
+ legacy_workgroup_regions = _restore_regional_store(
+ _workgroups, data.get("_workgroups", {})
+ )
+ for store, key, workgroup_fields in (
+ (_executions, "_executions", ("WorkGroup",)),
+ (_named_queries, "_named_queries", ("WorkGroup",)),
+ (
+ _prepared_statements,
+ "_prepared_statements",
+ ("WorkGroupName", "WorkGroup"),
+ ),
+ ):
+ _restore_workgroup_child_store(
+ store,
+ data.get(key, {}),
+ legacy_workgroup_regions,
+ workgroup_fields,
+ )
+ _restore_regional_store(_data_catalogs, data.get("_data_catalogs", {}))
_tags.update(data.get("_tags", {}))
+def _legacy_region_for_value(store, key, value):
+ """Infer an ARN region, falling back to Athena's configured boot region."""
+ request_region = get_region()
+ try:
+ set_request_region(REGION)
+ return store._region_for_legacy_value(key, value)
+ finally:
+ set_request_region(request_region)
+
+
+def _restore_regional_store(store, restored):
+ """Restore current state verbatim and legacy state deterministically."""
+ if isinstance(restored, AccountRegionScopedDict):
+ store.update(restored)
+ return {}
+
+ restored_regions = {}
+ if isinstance(restored, AccountScopedDict):
+ items = restored._data.items()
+ else:
+ account_id = get_account_id()
+ items = (((account_id, key), value) for key, value in restored.items())
+
+ for (account_id, key), value in items:
+ region = _legacy_region_for_value(store, key, value)
+ store.set_scoped(account_id, region, key, value)
+ restored_regions[(account_id, key)] = region
+ return restored_regions
+
+
+def _restore_workgroup_child_store(
+ store, restored, workgroup_regions, workgroup_fields
+):
+ """Place legacy child records beside their referenced workgroup."""
+ if isinstance(restored, AccountRegionScopedDict):
+ store.update(restored)
+ return
+
+ if isinstance(restored, AccountScopedDict):
+ items = restored._data.items()
+ else:
+ account_id = get_account_id()
+ items = (((account_id, key), value) for key, value in restored.items())
+
+ for (account_id, key), value in items:
+ workgroup = next(
+ (value.get(field) for field in workgroup_fields if value.get(field)),
+ None,
+ )
+ if not workgroup and store is _prepared_statements:
+ workgroup = str(key).partition("/")[0]
+ workgroup = workgroup or "primary"
+ region = workgroup_regions.get((account_id, workgroup))
+ if region is None:
+ region = _legacy_region_for_value(store, key, value)
+ store.set_scoped(account_id, region, key, value)
+
+
try:
_restored = load_state("athena")
if _restored:
diff --git a/tests/test_athena.py b/tests/test_athena.py
index b102e3ca..606287d8 100644
--- a/tests/test_athena.py
+++ b/tests/test_athena.py
@@ -6,10 +6,23 @@ import uuid as _uuid_mod
import zipfile
from urllib.parse import urlparse
+import boto3
import pytest
+from botocore.config import Config
from botocore.exceptions import ClientError
+def _client(region):
+ return boto3.client(
+ "athena",
+ endpoint_url=os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566"),
+ region_name=region,
+ aws_access_key_id="test",
+ aws_secret_access_key="test",
+ config=Config(retries={"mode": "standard"}),
+ )
+
+
def test_athena_query(athena):
resp = athena.start_query_execution(
QueryString="SELECT 1 AS num, 'hello' AS greeting",
@@ -91,6 +104,273 @@ def test_athena_workgroup_v2(athena):
with pytest.raises(ClientError):
athena.get_work_group(WorkGroup="ath-wg-v2")
+
+def test_athena_named_resources_are_region_scoped():
+ east = _client("us-east-1")
+ west = _client("us-west-2")
+ suffix = _uuid_mod.uuid4().hex[:8]
+ workgroup = f"ath-region-wg-{suffix}"
+ catalog = f"ath-region-catalog-{suffix}"
+ statement = f"ath-region-statement-{suffix}"
+
+ for client, region in ((east, "east"), (west, "west")):
+ client.create_work_group(Name=workgroup, Description=region)
+ client.create_data_catalog(Name=catalog, Type="HIVE", Description=region)
+ client.create_prepared_statement(
+ StatementName=statement,
+ WorkGroup=workgroup,
+ QueryStatement=f"SELECT '{region}'",
+ )
+
+ try:
+ assert east.get_work_group(WorkGroup=workgroup)["WorkGroup"][
+ "Description"
+ ] == "east"
+ assert west.get_work_group(WorkGroup=workgroup)["WorkGroup"][
+ "Description"
+ ] == "west"
+ assert east.get_data_catalog(Name=catalog)["DataCatalog"][
+ "Description"
+ ] == "east"
+ assert west.get_data_catalog(Name=catalog)["DataCatalog"][
+ "Description"
+ ] == "west"
+ assert east.get_prepared_statement(
+ StatementName=statement, WorkGroup=workgroup
+ )["PreparedStatement"]["QueryStatement"] == "SELECT 'east'"
+ assert west.get_prepared_statement(
+ StatementName=statement, WorkGroup=workgroup
+ )["PreparedStatement"]["QueryStatement"] == "SELECT 'west'"
+ finally:
+ for client in (east, west):
+ client.delete_prepared_statement(
+ StatementName=statement, WorkGroup=workgroup
+ )
+ client.delete_data_catalog(Name=catalog)
+ client.delete_work_group(
+ WorkGroup=workgroup, RecursiveDeleteOption=True
+ )
+
+
+def test_athena_defaults_are_seeded_per_region():
+ from ministack.core.responses import (
+ get_account_id,
+ get_region,
+ set_request_account_id,
+ set_request_region,
+ )
+ from ministack.services import athena as service
+
+ original_account = get_account_id()
+ original_region = get_region()
+ account_id = "111111111111"
+
+ service.reset()
+ try:
+ set_request_account_id(account_id)
+ set_request_region("us-east-1")
+ service._ensure_default_workgroup()
+ service._ensure_default_data_catalog()
+ service._workgroups["primary"]["Description"] = "east primary"
+
+ set_request_region("us-west-2")
+ service._ensure_default_workgroup()
+ service._ensure_default_data_catalog()
+
+ assert service._workgroups["primary"]["Description"] == "Primary workgroup"
+ assert service._data_catalogs["AwsDataCatalog"]["Type"] == "GLUE"
+ assert service._workgroups.get_scoped(
+ account_id, "us-east-1", "primary"
+ )["Description"] == "east primary"
+ assert service._data_catalogs.get_scoped(
+ account_id, "us-east-1", "AwsDataCatalog"
+ )["Type"] == "GLUE"
+ finally:
+ service.reset()
+ set_request_account_id(original_account)
+ set_request_region(original_region)
+
+
+def test_athena_legacy_state_migrates_to_configured_boot_region(monkeypatch):
+ from ministack.core.responses import (
+ AccountScopedDict,
+ get_account_id,
+ get_region,
+ set_request_account_id,
+ set_request_region,
+ )
+ from ministack.services import athena as service
+
+ original_account = get_account_id()
+ original_region = get_region()
+ account_id = "111111111111"
+ boot_region = "us-west-2"
+ first_request_region = "eu-central-1"
+ tags = AccountScopedDict()
+ stores = {
+ "_executions": ("legacy-execution", {"QueryExecutionId": "legacy-execution"}),
+ "_workgroups": ("legacy-workgroup", {"Name": "legacy-workgroup"}),
+ "_named_queries": ("legacy-query", {"NamedQueryId": "legacy-query"}),
+ "_data_catalogs": ("legacy-catalog", {"Name": "legacy-catalog"}),
+ "_prepared_statements": (
+ "legacy-workgroup/legacy-statement",
+ {"StatementName": "legacy-statement", "WorkGroup": "legacy-workgroup"},
+ ),
+ }
+
+ set_request_account_id(account_id)
+ monkeypatch.setattr(service, "REGION", boot_region)
+ set_request_region(first_request_region)
+ payload = {}
+ for store_name, (key, value) in stores.items():
+ legacy = AccountScopedDict()
+ legacy[key] = value
+ payload[store_name] = legacy
+ tag_arn = f"arn:aws:athena:{boot_region}:{account_id}:workgroup/legacy-workgroup"
+ tags[tag_arn] = {"legacy": "true"}
+ payload["_tags"] = tags
+
+ service.reset()
+ try:
+ service.restore_state(payload)
+ for store_name, (key, value) in stores.items():
+ restored = getattr(service, store_name).get_scoped(
+ account_id, boot_region, key
+ )
+ assert restored == value
+ assert (
+ getattr(service, store_name).get_scoped(
+ account_id, first_request_region, key
+ )
+ is None
+ )
+ assert service._tags.get(tag_arn) == {"legacy": "true"}
+ assert get_region() == first_request_region
+ finally:
+ service.reset()
+ set_request_account_id(original_account)
+ set_request_region(original_region)
+
+
+def test_athena_legacy_children_follow_workgroup_region(monkeypatch):
+ from ministack.core.responses import (
+ AccountScopedDict,
+ get_account_id,
+ get_region,
+ set_request_account_id,
+ set_request_region,
+ )
+ from ministack.services import athena as service
+
+ original_account = get_account_id()
+ original_region = get_region()
+ account_id = "111111111111"
+ boot_region = "us-east-1"
+ workgroup_region = "us-west-2"
+ first_request_region = "eu-central-1"
+ workgroup_name = "legacy-workgroup"
+
+ def legacy_store(key, value):
+ store = AccountScopedDict()
+ store._data[(account_id, key)] = value
+ return store
+
+ payload = {
+ "_workgroups": legacy_store(
+ workgroup_name,
+ {
+ "Name": workgroup_name,
+ "Configuration": {
+ "ResultConfiguration": {
+ "EncryptionConfiguration": {
+ "KmsKey": (
+ f"arn:aws:kms:{workgroup_region}:{account_id}:key/key-id"
+ )
+ }
+ }
+ },
+ },
+ ),
+ "_executions": legacy_store(
+ "legacy-execution",
+ {"QueryExecutionId": "legacy-execution", "WorkGroup": workgroup_name},
+ ),
+ "_named_queries": legacy_store(
+ "legacy-query",
+ {"NamedQueryId": "legacy-query", "WorkGroup": workgroup_name},
+ ),
+ "_prepared_statements": legacy_store(
+ f"{workgroup_name}/legacy-statement",
+ {
+ "StatementName": "legacy-statement",
+ "WorkGroupName": workgroup_name,
+ },
+ ),
+ "_data_catalogs": legacy_store(
+ "legacy-catalog", {"Name": "legacy-catalog"}
+ ),
+ }
+
+ monkeypatch.setattr(service, "REGION", boot_region)
+ set_request_account_id(account_id)
+ set_request_region(first_request_region)
+ service.reset()
+ try:
+ assert service.REGION != first_request_region
+ service.restore_state(payload)
+
+ assert service._workgroups.get_scoped(
+ account_id, workgroup_region, workgroup_name
+ )
+ for store_name, key in (
+ ("_executions", "legacy-execution"),
+ ("_named_queries", "legacy-query"),
+ ("_prepared_statements", f"{workgroup_name}/legacy-statement"),
+ ):
+ store = getattr(service, store_name)
+ assert store.get_scoped(account_id, workgroup_region, key)
+ assert store.get_scoped(account_id, boot_region, key) is None
+ assert store.get_scoped(account_id, first_request_region, key) is None
+ assert service._data_catalogs.get_scoped(
+ account_id, boot_region, "legacy-catalog"
+ )
+ assert get_region() == first_request_region
+ finally:
+ service.reset()
+ set_request_account_id(original_account)
+ set_request_region(original_region)
+
+
+def test_reset_clears_athena_state_across_regions():
+ from ministack.core.responses import get_region, set_request_region
+ from ministack.services import athena as service
+
+ original_region = get_region()
+ regional_stores = (
+ service._executions,
+ service._workgroups,
+ service._named_queries,
+ service._data_catalogs,
+ service._prepared_statements,
+ )
+
+ service.reset()
+ try:
+ for region in ("us-east-1", "us-west-2"):
+ set_request_region(region)
+ for store in regional_stores:
+ store[f"resource-{region}"] = {"region": region}
+ service._tags["arn:aws:athena:us-east-1:000000000000:workgroup/tagged"] = {
+ "tag": "value"
+ }
+
+ service.reset()
+ assert all(not store.has_any() for store in regional_stores)
+ assert not service._tags._data
+ finally:
+ service.reset()
+ set_request_region(original_region)
+
def test_athena_named_query_v2(athena):
resp = athena.create_named_query(
Name="ath-nq-v2",
diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index 2d8a2eca..3de3fde3 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -1832,6 +1832,39 @@ def test_ecs_region_scoped_state_is_rejected_by_v2_reader(monkeypatch, tmp_path)
assert persistence.load_state("ecs") is None
+def test_athena_region_scoped_state_is_rejected_by_v2_reader(
+ monkeypatch, tmp_path
+):
+ """A rollback binary must reject Athena's regional schema instead of
+ accepting it as v2 and silently dropping every regional store."""
+ import json as _json
+
+ from ministack.core.responses import AccountRegionScopedDict
+
+ monkeypatch.setattr(persistence, "PERSIST_STATE", True)
+ monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
+
+ workgroups = AccountRegionScopedDict()
+ workgroups.set_scoped(
+ "000000000000",
+ "us-west-2",
+ "regional-workgroup",
+ {"Name": "regional-workgroup", "Description": "west"},
+ )
+ persistence.save_state("athena", {"_workgroups": workgroups})
+
+ raw = _json.loads((tmp_path / "athena.json").read_text())
+ assert raw["__ministack_format__"] == 3
+ loaded_workgroups = persistence.load_state("athena")["_workgroups"]
+ assert loaded_workgroups.get_scoped(
+ "000000000000", "us-west-2", "regional-workgroup"
+ )["Description"] == "west"
+
+ # Simulate the previous binary, whose highest understood format is v2.
+ monkeypatch.setattr(persistence, "SERVICE_STATE_FORMAT_VERSIONS", {})
+ assert persistence.load_state("athena") is None
+
+
def test_resource_groups_region_scoped_state_is_rejected_by_v2_reader(
monkeypatch, tmp_path
):