Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
Scope Scheduler state by region (#1110)
Ian Oberst444890b15da1
diff --git a/ministack/services/scheduler.py b/ministack/services/scheduler.pyindex 61e8f39a..59315d51 100644--- a/ministack/services/scheduler.py+++ b/ministack/services/scheduler.py@@ -23,11 +23,13 @@ from urllib.parse import unquotefrom ministack.core.arn import ArnParseError, parse_arnfrom ministack.core.persistence import load_statefrom ministack.core.responses import (+ AccountRegionScopedDict,AccountScopedDict,get_account_id,get_region,new_uuid,set_request_account_id,+ set_request_region,)logger = logging.getLogger("scheduler")@@ -38,9 +40,9 @@ REGION = os.environ.get("MINISTACK_REGION", "us-east-1")# State# ----------------------------------------------------------------------------_schedules = AccountScopedDict() # (group, name) -> schedule record-_schedule_groups = AccountScopedDict() # group_name -> group record-_tags = AccountScopedDict() # arn -> {key: value}+_schedules = AccountRegionScopedDict() # "group/name" -> schedule record+_schedule_groups = AccountRegionScopedDict() # group_name -> group record+_tags = AccountScopedDict() # ARN key embeds region; account scoping is sufficient.def reset():@@ -50,10 +52,9 @@ def reset():def get_state():- # Preserve AccountScopedDict wrappers; casting to a plain dict drops the- # per-account scoping and would persist only the current request's- # tenants. AccountScopedDict has a JSON encoder hook in core/persistence- # that round-trips the (account, key) tuple correctly.+ # Preserve scoped wrappers; casting to a plain dict drops account/region+ # metadata and would persist only the current request's resources. Both+ # wrapper types have JSON encoder hooks in core/persistence.return {"schedules": copy.deepcopy(_schedules),"schedule_groups": copy.deepcopy(_schedule_groups),@@ -478,6 +479,8 @@ async def handle_request(method, path, headers, body_bytes, query_params):if method == "DELETE":return _untag_resource(arn, query)+ return _error(400, "ValidationException", f"No route for {method} {path}")+# ---------------------------------------------------------------------------# Schedule firing (issue #958)@@ -490,7 +493,7 @@ async def handle_request(method, path, headers, body_bytes, query_params):# ``ActionAfterCompletion`` one-shot delete, ``State`` and ``StartDate``/``EndDate``._SCHEDULE_TICK_INTERVAL = 10 # seconds between sweeps-_schedule_last_fired: dict = {} # (account_id, (group, name)) -> epoch; not persisted+_schedule_last_fired: dict = {} # (account_id, region, "group/name") -> epoch; not persisted_ticker_thread: "threading.Thread | None" = None@@ -529,78 +532,91 @@ def _tick_schedules():now = time.time()now_dt = datetime.fromtimestamp(now, tz=timezone.utc)- for state_key, sched in list(_schedules._data.items()):- account_id, key = state_key- if sched.get("State") != "ENABLED":- continue- end = _to_epoch(sched.get("EndDate"))- if end is not None and now > end:- # A recurring schedule past its EndDate has "completed".- if sched.get("ActionAfterCompletion") == "DELETE":- _schedules._data.pop(state_key, None)- _schedule_last_fired.pop(state_key, None)- continue- start = _to_epoch(sched.get("StartDate"))- if start is not None and now < start:- continue+ previous_account = get_account_id()+ previous_region = get_region()+ try:+ for state_key, sched in list(_schedules._data.items()):+ account_id, region, key = state_key+ if sched.get("State") != "ENABLED":+ continue+ end = _to_epoch(sched.get("EndDate"))+ if end is not None and now > end:+ # A recurring schedule past its EndDate has "completed".+ if sched.get("ActionAfterCompletion") == "DELETE":+ _schedules._data.pop(state_key, None)+ _schedule_last_fired.pop(state_key, None)+ continue+ start = _to_epoch(sched.get("StartDate"))+ if start is not None and now < start:+ continue- expr = sched.get("ScheduleExpression", "")- one_shot = False+ expr = sched.get("ScheduleExpression", "")+ one_shot = False- at_epoch = _at_time_epoch(expr)- if at_epoch is not None:- # One-time at(): fire once when the moment has passed.- if now < at_epoch or state_key in _schedule_last_fired:- continue- one_shot = True- else:- interval = _eb._parse_rate_seconds(expr)- if interval is not None:- # rate(): countdown anchored to creation, then every interval.- if state_key not in _schedule_last_fired:- _schedule_last_fired[state_key] = sched.get("CreationDate", now)- if now - _schedule_last_fired[state_key] < interval:+ at_epoch = _at_time_epoch(expr)+ if at_epoch is not None:+ # One-time at(): fire once when the moment has passed.+ if now < at_epoch or state_key in _schedule_last_fired:continue+ one_shot = Trueelse:- fields = _eb._parse_cron_fields(expr)- if fields is None:- continue # unsupported / unparseable expression- # cron(): fire once per scheduled occurrence.- if state_key not in _schedule_last_fired:- _schedule_last_fired[state_key] = now- continue- last_dt = datetime.fromtimestamp(_schedule_last_fired[state_key], tz=timezone.utc)- next_fire = _eb._cron_next_fire(fields, last_dt)- if next_fire is None or now_dt < next_fire:- continue-- _schedule_last_fired[state_key] = now- target = sched.get("Target") or {}- if not target.get("Arn"):- continue- # Fire under the schedule's own tenant so ARN-building / dispatch scope right.- set_request_account_id(account_id)- event = {- "EventId": new_uuid(),- "Source": "aws.scheduler",- "DetailType": "Scheduled Event",- "Detail": "{}",- "Time": now,- "Resources": [sched.get("Arn", "")],- "Account": account_id,- "Region": get_region(),- }- try:- _eb._invoke_target(target, event, sched)- except Exception:- logger.exception("Scheduler dispatch error for %s (account %s)", key, account_id)-- # ActionAfterCompletion=DELETE on a one-time at() schedule: remove after- # firing. Recurring DELETE schedules "complete" at EndDate (handled above);- # at() with NONE stays but never refires (guarded by _schedule_last_fired).- if one_shot and sched.get("ActionAfterCompletion") == "DELETE":- _schedules._data.pop(state_key, None)- _schedule_last_fired.pop(state_key, None)+ interval = _eb._parse_rate_seconds(expr)+ if interval is not None:+ # rate(): countdown anchored to creation, then every interval.+ if state_key not in _schedule_last_fired:+ _schedule_last_fired[state_key] = sched.get("CreationDate", now)+ if now - _schedule_last_fired[state_key] < interval:+ continue+ else:+ fields = _eb._parse_cron_fields(expr)+ if fields is None:+ continue # unsupported / unparseable expression+ # cron(): fire once per scheduled occurrence.+ if state_key not in _schedule_last_fired:+ _schedule_last_fired[state_key] = now+ continue+ last_dt = datetime.fromtimestamp(_schedule_last_fired[state_key], tz=timezone.utc)+ next_fire = _eb._cron_next_fire(fields, last_dt)+ if next_fire is None or now_dt < next_fire:+ continue++ _schedule_last_fired[state_key] = now+ target = sched.get("Target") or {}+ if not target.get("Arn"):+ continue+ # Fire under the schedule's own account and region so target dispatch+ # observes the same request scope as the schedule resource.+ set_request_account_id(account_id)+ set_request_region(region)+ event = {+ "EventId": new_uuid(),+ "Source": "aws.scheduler",+ "DetailType": "Scheduled Event",+ "Detail": "{}",+ "Time": now,+ "Resources": [sched.get("Arn", "")],+ "Account": account_id,+ "Region": get_region(),+ }+ try:+ _eb._invoke_target(target, event, sched)+ except Exception:+ logger.exception(+ "Scheduler dispatch error for %s (account %s, region %s)",+ key,+ account_id,+ region,+ )++ # ActionAfterCompletion=DELETE on a one-time at() schedule: remove after+ # firing. Recurring DELETE schedules "complete" at EndDate (handled above);+ # at() with NONE stays but never refires (guarded by _schedule_last_fired).+ if one_shot and sched.get("ActionAfterCompletion") == "DELETE":+ _schedules._data.pop(state_key, None)+ _schedule_last_fired.pop(state_key, None)+ finally:+ set_request_account_id(previous_account)+ set_request_region(previous_region)def _ticker_loop():@@ -622,5 +638,3 @@ def start_scheduler() -> None:target=_ticker_loop, daemon=True, name="evb-scheduler-ticker")_ticker_thread.start()-- return _error(400, "ValidationException", f"No route for {method} {path}")diff --git a/tests/test_persistence.py b/tests/test_persistence.pyindex 29a6f93a..ac978d82 100644--- a/tests/test_persistence.py+++ b/tests/test_persistence.py@@ -506,15 +506,90 @@ def test_scheduler_round_trip():# pre-existing inline comment on the dict mis-describes the shape. Use# the real production key shape so this test catches a regression that# broke string-key serialisation.+ from ministack.core.responses import get_region, set_request_region++ original_region = get_region()+def populate(mod):- mod._schedule_groups["default"] = {"Name": "default"}- mod._schedules["default/sched-test"] = {"Name": "sched-test"}+ set_request_region("us-east-1")+ mod._schedule_groups["default"] = {+ "Arn": "arn:aws:scheduler:us-east-1:000000000000:schedule-group/default",+ "Name": "default",+ }+ mod._schedules["default/sched-test"] = {+ "Arn": "arn:aws:scheduler:us-east-1:000000000000:schedule/default/sched-test",+ "Name": "sched-test",+ "ScheduleExpression": "rate(1 hour)",+ }+ set_request_region("us-west-2")+ mod._schedule_groups["default"] = {+ "Arn": "arn:aws:scheduler:us-west-2:000000000000:schedule-group/default",+ "Name": "default",+ }+ mod._schedules["default/sched-test"] = {+ "Arn": "arn:aws:scheduler:us-west-2:000000000000:schedule/default/sched-test",+ "Name": "sched-test",+ "ScheduleExpression": "rate(2 hours)",+ }def observe(mod):- assert "default" in mod._schedule_groups- assert "default/sched-test" in mod._schedules+ set_request_region("us-east-1")+ assert mod._schedules["default/sched-test"]["ScheduleExpression"] == "rate(1 hour)"+ set_request_region("us-west-2")+ assert mod._schedules["default/sched-test"]["ScheduleExpression"] == "rate(2 hours)"++ try:+ _round_trip("scheduler", "scheduler", populate, observe)+ finally:+ set_request_region(original_region)+++def test_scheduler_legacy_account_scoped_state_uses_resource_arn_region():+ from ministack.core.responses import (+ AccountScopedDict,+ get_account_id,+ get_region,+ set_request_account_id,+ set_request_region,+ )+ from ministack.services import scheduler as mod- _round_trip("scheduler", "scheduler", populate, observe)+ original_account = get_account_id()+ original_region = get_region()+ account_id = "000000000000"+ region = "us-west-2"+ group = "legacy-group"+ schedule_key = f"{group}/legacy-schedule"+ legacy_groups = AccountScopedDict()+ legacy_groups._data[(account_id, group)] = {+ "Arn": f"arn:aws:scheduler:{region}:{account_id}:schedule-group/{group}",+ "Name": group,+ }+ legacy_schedules = AccountScopedDict()+ legacy_schedules._data[(account_id, schedule_key)] = {+ "Arn": f"arn:aws:scheduler:{region}:{account_id}:schedule/{schedule_key}",+ "Name": "legacy-schedule",+ "GroupName": group,+ }++ mod.reset()+ try:+ set_request_account_id(account_id)+ set_request_region("us-east-1")+ mod.restore_state(+ {"schedule_groups": legacy_groups, "schedules": legacy_schedules}+ )++ assert mod._schedule_groups.get_scoped(account_id, "us-east-1", group) is None+ assert mod._schedules.get_scoped(account_id, "us-east-1", schedule_key) is None+ assert mod._schedule_groups.get_scoped(account_id, region, group)["Name"] == group+ assert mod._schedules.get_scoped(account_id, region, schedule_key)["Name"] == (+ "legacy-schedule"+ )+ finally:+ mod.reset()+ set_request_account_id(original_account)+ set_request_region(original_region)def test_pipes_round_trip():diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex c90d2394..e5f6230b 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -9,11 +9,19 @@ ENDPOINT = "http://localhost:4566"REGION = "us-east-1"+def _scheduler_client(region):+ return boto3.client(+ "scheduler",+ endpoint_url=ENDPOINT,+ aws_access_key_id="test",+ aws_secret_access_key="test",+ region_name=region,+ )++@pytest.fixture(scope="module")def scheduler():- return boto3.client("scheduler", endpoint_url=ENDPOINT,- aws_access_key_id="test", aws_secret_access_key="test",- region_name=REGION)+ return _scheduler_client(REGION)@pytest.fixture(scope="module")@@ -60,6 +68,105 @@ def test_scheduler_create_get_delete_group(scheduler):assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"+def test_scheduler_groups_schedules_and_tags_are_region_scoped(scheduler):+ west = _scheduler_client("us-west-2")+ group = f"region-group-{_uid()}"+ name = f"region-schedule-{_uid()}"++ east_group_arn = scheduler.create_schedule_group(Name=group)["ScheduleGroupArn"]+ west_group_arn = west.create_schedule_group(Name=group)["ScheduleGroupArn"]+ assert east_group_arn == f"arn:aws:scheduler:us-east-1:000000000000:schedule-group/{group}"+ assert west_group_arn == f"arn:aws:scheduler:us-west-2:000000000000:schedule-group/{group}"+ assert [item["Arn"] for item in scheduler.list_schedule_groups(NamePrefix=group)["ScheduleGroups"]] == [+ east_group_arn+ ]+ assert [item["Arn"] for item in west.list_schedule_groups(NamePrefix=group)["ScheduleGroups"]] == [+ west_group_arn+ ]++ east_schedule_arn = scheduler.create_schedule(+ Name=name,+ GroupName=group,+ ScheduleExpression="rate(1 hour)",+ FlexibleTimeWindow={"Mode": "OFF"},+ Target={+ "Arn": "arn:aws:lambda:us-east-1:000000000000:function:east-target",+ "RoleArn": "arn:aws:iam::000000000000:role/test",+ },+ )["ScheduleArn"]+ west_schedule_arn = west.create_schedule(+ Name=name,+ GroupName=group,+ ScheduleExpression="rate(2 hours)",+ FlexibleTimeWindow={"Mode": "OFF"},+ Target={+ "Arn": "arn:aws:lambda:us-west-2:000000000000:function:west-target",+ "RoleArn": "arn:aws:iam::000000000000:role/test",+ },+ )["ScheduleArn"]+ assert east_schedule_arn == f"arn:aws:scheduler:us-east-1:000000000000:schedule/{group}/{name}"+ assert west_schedule_arn == f"arn:aws:scheduler:us-west-2:000000000000:schedule/{group}/{name}"++ assert scheduler.get_schedule(Name=name, GroupName=group)["ScheduleExpression"] == "rate(1 hour)"+ assert west.get_schedule(Name=name, GroupName=group)["ScheduleExpression"] == "rate(2 hours)"+ assert [item["Arn"] for item in scheduler.list_schedules(GroupName=group)["Schedules"]] == [+ east_schedule_arn+ ]+ assert [item["Arn"] for item in west.list_schedules(GroupName=group)["Schedules"]] == [+ west_schedule_arn+ ]++ scheduler.tag_resource(ResourceArn=east_schedule_arn, Tags=[{"Key": "region", "Value": "east"}])+ west.tag_resource(ResourceArn=west_schedule_arn, Tags=[{"Key": "region", "Value": "west"}])+ assert scheduler.list_tags_for_resource(ResourceArn=east_schedule_arn)["Tags"] == [+ {"Key": "region", "Value": "east"}+ ]+ assert west.list_tags_for_resource(ResourceArn=west_schedule_arn)["Tags"] == [+ {"Key": "region", "Value": "west"}+ ]++ scheduler.update_schedule(+ Name=name,+ GroupName=group,+ ScheduleExpression="rate(3 hours)",+ FlexibleTimeWindow={"Mode": "OFF"},+ Target={+ "Arn": "arn:aws:lambda:us-east-1:000000000000:function:east-target-updated",+ "RoleArn": "arn:aws:iam::000000000000:role/test",+ },+ )+ assert scheduler.get_schedule(Name=name, GroupName=group)["ScheduleExpression"] == "rate(3 hours)"+ assert west.get_schedule(Name=name, GroupName=group)["ScheduleExpression"] == "rate(2 hours)"++ scheduler.delete_schedule(Name=name, GroupName=group)+ with pytest.raises(ClientError) as exc:+ scheduler.get_schedule(Name=name, GroupName=group)+ assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"+ with pytest.raises(ClientError) as exc:+ scheduler.update_schedule(+ Name=name,+ GroupName=group,+ ScheduleExpression="rate(4 hours)",+ FlexibleTimeWindow={"Mode": "OFF"},+ Target={+ "Arn": "arn:aws:lambda:us-east-1:000000000000:function:missing",+ "RoleArn": "arn:aws:iam::000000000000:role/test",+ },+ )+ assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"+ with pytest.raises(ClientError) as exc:+ scheduler.delete_schedule(Name=name, GroupName=group)+ assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"++ scheduler.delete_schedule_group(Name=group)+ assert west.get_schedule_group(Name=group)["Arn"] == west_group_arn+ assert west.get_schedule(Name=name, GroupName=group)["Arn"] == west_schedule_arn+ assert west.list_tags_for_resource(ResourceArn=west_schedule_arn)["Tags"] == [+ {"Key": "region", "Value": "west"}+ ]+ west.delete_schedule_group(Name=group)++def test_scheduler_create_duplicate_group(scheduler):name = f"dup-group-{_uid()}"scheduler.create_schedule_group(Name=name)diff --git a/tests/test_scheduler_arn_adoption.py b/tests/test_scheduler_arn_adoption.pyindex f8d2ff3f..9467e0cb 100644--- a/tests/test_scheduler_arn_adoption.py+++ b/tests/test_scheduler_arn_adoption.py@@ -5,7 +5,12 @@ import uuidimport pytestimport ministack.services.scheduler as scheduler-from ministack.core.responses import set_request_account_id, set_request_region+from ministack.core.responses import (+ get_account_id,+ get_region,+ set_request_account_id,+ set_request_region,+)def _uid():@@ -17,8 +22,12 @@ def _reset_scheduler():set_request_account_id("000000000000")set_request_region("us-east-1")scheduler.reset()+ scheduler._schedule_last_fired.clear()+ scheduler._ticker_thread = Noneyieldscheduler.reset()+ scheduler._schedule_last_fired.clear()+ scheduler._ticker_thread = Nonedef _request(method, path, body=None, query=None):@@ -114,6 +123,95 @@ def test_scheduler_tag_apis_do_not_resolve_same_named_resources_from_other_regioassert scheduler._tags.get(west_group_arn) is None+def test_scheduler_ticker_dispatches_in_schedule_region_and_restores_context(monkeypatch):+ from ministack.services import eventbridge++ account_id = "000000000000"+ name = f"ticker-{_uid()}"+ key = f"default/{name}"+ expected = {}+ for region in ("us-east-1", "us-west-2"):+ schedule_arn = f"arn:aws:scheduler:{region}:{account_id}:schedule/{key}"+ target_arn = f"arn:aws:lambda:{region}:{account_id}:function:ticker-target"+ expected[region] = (schedule_arn, target_arn)+ scheduler._schedules.set_scoped(+ account_id,+ region,+ key,+ {+ "Arn": schedule_arn,+ "Name": name,+ "GroupName": "default",+ "ScheduleExpression": "at(1970-01-01T00:00:01)",+ "Target": {"Arn": target_arn},+ "State": "ENABLED",+ "ActionAfterCompletion": "NONE",+ "CreationDate": 1,+ },+ )+ set_request_account_id("111111111111")+ set_request_region("eu-west-1")++ calls = []++ def _capture_target(target, event, schedule):+ calls.append((get_account_id(), get_region(), target, event, schedule))++ monkeypatch.setattr(eventbridge, "_invoke_target", _capture_target)+ monkeypatch.setattr(scheduler.time, "time", lambda: 2.0)++ scheduler._tick_schedules()++ assert len(calls) == 2+ calls_by_region = {call[1]: call for call in calls}+ assert set(calls_by_region) == set(expected)+ for region, (schedule_arn, target_arn) in expected.items():+ dispatch_account, dispatch_region, target, event, schedule = calls_by_region[region]+ assert (dispatch_account, dispatch_region) == (account_id, region)+ assert target["Arn"] == target_arn+ assert event["Account"] == account_id+ assert event["Region"] == region+ assert event["Resources"] == [schedule_arn]+ assert schedule["Arn"] == schedule_arn+ assert (get_account_id(), get_region()) == ("111111111111", "eu-west-1")+++def test_scheduler_start_scheduler_starts_daemon_once(monkeypatch):+ created_threads = []++ class _FakeThread:+ def __init__(self, *, target, daemon, name):+ self.target = target+ self.daemon = daemon+ self.name = name+ self.started = False+ created_threads.append(self)++ def start(self):+ self.started = True++ def is_alive(self):+ return self.started++ monkeypatch.setattr(scheduler.threading, "Thread", _FakeThread)++ scheduler.start_scheduler()+ scheduler.start_scheduler()++ assert len(created_threads) == 1+ thread = created_threads[0]+ assert thread.target is scheduler._ticker_loop+ assert thread.daemon is True+ assert thread.name == "evb-scheduler-ticker"+ assert thread.started is True+++def test_scheduler_unknown_route_returns_validation_error():+ response = _request("GET", "/unknown")++ _assert_error(response, 400, "ValidationException")++@pytest.mark.parametrize("bad_arn, expected_status, expected_code",[