Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

Scope Scheduler state by region (#1110)

Ian Oberst444890b15da1
diff --git a/ministack/services/scheduler.py b/ministack/services/scheduler.py
index 61e8f39a..59315d51 100644
--- a/ministack/services/scheduler.py
+++ b/ministack/services/scheduler.py
@@ -23,11 +23,13 @@ from urllib.parse import unquote
from ministack.core.arn import ArnParseError, parse_arn
from ministack.core.persistence import load_state
from ministack.core.responses import (
+ AccountRegionScopedDict,
AccountScopedDict,
get_account_id,
get_region,
new_uuid,
set_request_account_id,
+ set_request_region,
)
logger = logging.getLogger("scheduler")
@@ -38,9 +40,9 @@ REGION = os.environ.get("MINISTACK_REGION", "us-east-1")
# State
# ---------------------------------------------------------------------------
-_schedules = AccountScopedDict() # (group, name) -> schedule record
-_schedule_groups = AccountScopedDict() # group_name -> group record
-_tags = AccountScopedDict() # arn -> {key: value}
+_schedules = AccountRegionScopedDict() # "group/name" -> schedule record
+_schedule_groups = AccountRegionScopedDict() # group_name -> group record
+_tags = AccountScopedDict() # ARN key embeds region; account scoping is sufficient.
def reset():
@@ -50,10 +52,9 @@ def reset():
def get_state():
- # Preserve AccountScopedDict wrappers; casting to a plain dict drops the
- # per-account scoping and would persist only the current request's
- # tenants. AccountScopedDict has a JSON encoder hook in core/persistence
- # that round-trips the (account, key) tuple correctly.
+ # Preserve scoped wrappers; casting to a plain dict drops account/region
+ # metadata and would persist only the current request's resources. Both
+ # wrapper types have JSON encoder hooks in core/persistence.
return {
"schedules": copy.deepcopy(_schedules),
"schedule_groups": copy.deepcopy(_schedule_groups),
@@ -478,6 +479,8 @@ async def handle_request(method, path, headers, body_bytes, query_params):
if method == "DELETE":
return _untag_resource(arn, query)
+ return _error(400, "ValidationException", f"No route for {method} {path}")
+
# ---------------------------------------------------------------------------
# Schedule firing (issue #958)
@@ -490,7 +493,7 @@ async def handle_request(method, path, headers, body_bytes, query_params):
# ``ActionAfterCompletion`` one-shot delete, ``State`` and ``StartDate``/``EndDate``.
_SCHEDULE_TICK_INTERVAL = 10 # seconds between sweeps
-_schedule_last_fired: dict = {} # (account_id, (group, name)) -> epoch; not persisted
+_schedule_last_fired: dict = {} # (account_id, region, "group/name") -> epoch; not persisted
_ticker_thread: "threading.Thread | None" = None
@@ -529,78 +532,91 @@ def _tick_schedules():
now = time.time()
now_dt = datetime.fromtimestamp(now, tz=timezone.utc)
- for state_key, sched in list(_schedules._data.items()):
- account_id, key = state_key
- if sched.get("State") != "ENABLED":
- continue
- end = _to_epoch(sched.get("EndDate"))
- if end is not None and now > end:
- # A recurring schedule past its EndDate has "completed".
- if sched.get("ActionAfterCompletion") == "DELETE":
- _schedules._data.pop(state_key, None)
- _schedule_last_fired.pop(state_key, None)
- continue
- start = _to_epoch(sched.get("StartDate"))
- if start is not None and now < start:
- continue
+ previous_account = get_account_id()
+ previous_region = get_region()
+ try:
+ for state_key, sched in list(_schedules._data.items()):
+ account_id, region, key = state_key
+ if sched.get("State") != "ENABLED":
+ continue
+ end = _to_epoch(sched.get("EndDate"))
+ if end is not None and now > end:
+ # A recurring schedule past its EndDate has "completed".
+ if sched.get("ActionAfterCompletion") == "DELETE":
+ _schedules._data.pop(state_key, None)
+ _schedule_last_fired.pop(state_key, None)
+ continue
+ start = _to_epoch(sched.get("StartDate"))
+ if start is not None and now < start:
+ continue
- expr = sched.get("ScheduleExpression", "")
- one_shot = False
+ expr = sched.get("ScheduleExpression", "")
+ one_shot = False
- at_epoch = _at_time_epoch(expr)
- if at_epoch is not None:
- # One-time at(): fire once when the moment has passed.
- if now < at_epoch or state_key in _schedule_last_fired:
- continue
- one_shot = True
- else:
- interval = _eb._parse_rate_seconds(expr)
- if interval is not None:
- # rate(): countdown anchored to creation, then every interval.
- if state_key not in _schedule_last_fired:
- _schedule_last_fired[state_key] = sched.get("CreationDate", now)
- if now - _schedule_last_fired[state_key] < interval:
+ at_epoch = _at_time_epoch(expr)
+ if at_epoch is not None:
+ # One-time at(): fire once when the moment has passed.
+ if now < at_epoch or state_key in _schedule_last_fired:
continue
+ one_shot = True
else:
- fields = _eb._parse_cron_fields(expr)
- if fields is None:
- continue # unsupported / unparseable expression
- # cron(): fire once per scheduled occurrence.
- if state_key not in _schedule_last_fired:
- _schedule_last_fired[state_key] = now
- continue
- last_dt = datetime.fromtimestamp(_schedule_last_fired[state_key], tz=timezone.utc)
- next_fire = _eb._cron_next_fire(fields, last_dt)
- if next_fire is None or now_dt < next_fire:
- continue
-
- _schedule_last_fired[state_key] = now
- target = sched.get("Target") or {}
- if not target.get("Arn"):
- continue
- # Fire under the schedule's own tenant so ARN-building / dispatch scope right.
- set_request_account_id(account_id)
- event = {
- "EventId": new_uuid(),
- "Source": "aws.scheduler",
- "DetailType": "Scheduled Event",
- "Detail": "{}",
- "Time": now,
- "Resources": [sched.get("Arn", "")],
- "Account": account_id,
- "Region": get_region(),
- }
- try:
- _eb._invoke_target(target, event, sched)
- except Exception:
- logger.exception("Scheduler dispatch error for %s (account %s)", key, account_id)
-
- # ActionAfterCompletion=DELETE on a one-time at() schedule: remove after
- # firing. Recurring DELETE schedules "complete" at EndDate (handled above);
- # at() with NONE stays but never refires (guarded by _schedule_last_fired).
- if one_shot and sched.get("ActionAfterCompletion") == "DELETE":
- _schedules._data.pop(state_key, None)
- _schedule_last_fired.pop(state_key, None)
+ interval = _eb._parse_rate_seconds(expr)
+ if interval is not None:
+ # rate(): countdown anchored to creation, then every interval.
+ if state_key not in _schedule_last_fired:
+ _schedule_last_fired[state_key] = sched.get("CreationDate", now)
+ if now - _schedule_last_fired[state_key] < interval:
+ continue
+ else:
+ fields = _eb._parse_cron_fields(expr)
+ if fields is None:
+ continue # unsupported / unparseable expression
+ # cron(): fire once per scheduled occurrence.
+ if state_key not in _schedule_last_fired:
+ _schedule_last_fired[state_key] = now
+ continue
+ last_dt = datetime.fromtimestamp(_schedule_last_fired[state_key], tz=timezone.utc)
+ next_fire = _eb._cron_next_fire(fields, last_dt)
+ if next_fire is None or now_dt < next_fire:
+ continue
+
+ _schedule_last_fired[state_key] = now
+ target = sched.get("Target") or {}
+ if not target.get("Arn"):
+ continue
+ # Fire under the schedule's own account and region so target dispatch
+ # observes the same request scope as the schedule resource.
+ set_request_account_id(account_id)
+ set_request_region(region)
+ event = {
+ "EventId": new_uuid(),
+ "Source": "aws.scheduler",
+ "DetailType": "Scheduled Event",
+ "Detail": "{}",
+ "Time": now,
+ "Resources": [sched.get("Arn", "")],
+ "Account": account_id,
+ "Region": get_region(),
+ }
+ try:
+ _eb._invoke_target(target, event, sched)
+ except Exception:
+ logger.exception(
+ "Scheduler dispatch error for %s (account %s, region %s)",
+ key,
+ account_id,
+ region,
+ )
+
+ # ActionAfterCompletion=DELETE on a one-time at() schedule: remove after
+ # firing. Recurring DELETE schedules "complete" at EndDate (handled above);
+ # at() with NONE stays but never refires (guarded by _schedule_last_fired).
+ if one_shot and sched.get("ActionAfterCompletion") == "DELETE":
+ _schedules._data.pop(state_key, None)
+ _schedule_last_fired.pop(state_key, None)
+ finally:
+ set_request_account_id(previous_account)
+ set_request_region(previous_region)
def _ticker_loop():
@@ -622,5 +638,3 @@ def start_scheduler() -> None:
target=_ticker_loop, daemon=True, name="evb-scheduler-ticker"
)
_ticker_thread.start()
-
- return _error(400, "ValidationException", f"No route for {method} {path}")
diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index 29a6f93a..ac978d82 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -506,15 +506,90 @@ def test_scheduler_round_trip():
# pre-existing inline comment on the dict mis-describes the shape. Use
# the real production key shape so this test catches a regression that
# broke string-key serialisation.
+ from ministack.core.responses import get_region, set_request_region
+
+ original_region = get_region()
+
def populate(mod):
- mod._schedule_groups["default"] = {"Name": "default"}
- mod._schedules["default/sched-test"] = {"Name": "sched-test"}
+ set_request_region("us-east-1")
+ mod._schedule_groups["default"] = {
+ "Arn": "arn:aws:scheduler:us-east-1:000000000000:schedule-group/default",
+ "Name": "default",
+ }
+ mod._schedules["default/sched-test"] = {
+ "Arn": "arn:aws:scheduler:us-east-1:000000000000:schedule/default/sched-test",
+ "Name": "sched-test",
+ "ScheduleExpression": "rate(1 hour)",
+ }
+ set_request_region("us-west-2")
+ mod._schedule_groups["default"] = {
+ "Arn": "arn:aws:scheduler:us-west-2:000000000000:schedule-group/default",
+ "Name": "default",
+ }
+ mod._schedules["default/sched-test"] = {
+ "Arn": "arn:aws:scheduler:us-west-2:000000000000:schedule/default/sched-test",
+ "Name": "sched-test",
+ "ScheduleExpression": "rate(2 hours)",
+ }
def observe(mod):
- assert "default" in mod._schedule_groups
- assert "default/sched-test" in mod._schedules
+ set_request_region("us-east-1")
+ assert mod._schedules["default/sched-test"]["ScheduleExpression"] == "rate(1 hour)"
+ set_request_region("us-west-2")
+ assert mod._schedules["default/sched-test"]["ScheduleExpression"] == "rate(2 hours)"
+
+ try:
+ _round_trip("scheduler", "scheduler", populate, observe)
+ finally:
+ set_request_region(original_region)
+
+
+def test_scheduler_legacy_account_scoped_state_uses_resource_arn_region():
+ from ministack.core.responses import (
+ AccountScopedDict,
+ get_account_id,
+ get_region,
+ set_request_account_id,
+ set_request_region,
+ )
+ from ministack.services import scheduler as mod
- _round_trip("scheduler", "scheduler", populate, observe)
+ original_account = get_account_id()
+ original_region = get_region()
+ account_id = "000000000000"
+ region = "us-west-2"
+ group = "legacy-group"
+ schedule_key = f"{group}/legacy-schedule"
+ legacy_groups = AccountScopedDict()
+ legacy_groups._data[(account_id, group)] = {
+ "Arn": f"arn:aws:scheduler:{region}:{account_id}:schedule-group/{group}",
+ "Name": group,
+ }
+ legacy_schedules = AccountScopedDict()
+ legacy_schedules._data[(account_id, schedule_key)] = {
+ "Arn": f"arn:aws:scheduler:{region}:{account_id}:schedule/{schedule_key}",
+ "Name": "legacy-schedule",
+ "GroupName": group,
+ }
+
+ mod.reset()
+ try:
+ set_request_account_id(account_id)
+ set_request_region("us-east-1")
+ mod.restore_state(
+ {"schedule_groups": legacy_groups, "schedules": legacy_schedules}
+ )
+
+ assert mod._schedule_groups.get_scoped(account_id, "us-east-1", group) is None
+ assert mod._schedules.get_scoped(account_id, "us-east-1", schedule_key) is None
+ assert mod._schedule_groups.get_scoped(account_id, region, group)["Name"] == group
+ assert mod._schedules.get_scoped(account_id, region, schedule_key)["Name"] == (
+ "legacy-schedule"
+ )
+ finally:
+ mod.reset()
+ set_request_account_id(original_account)
+ set_request_region(original_region)
def test_pipes_round_trip():
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index c90d2394..e5f6230b 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -9,11 +9,19 @@ ENDPOINT = "http://localhost:4566"
REGION = "us-east-1"
+def _scheduler_client(region):
+ return boto3.client(
+ "scheduler",
+ endpoint_url=ENDPOINT,
+ aws_access_key_id="test",
+ aws_secret_access_key="test",
+ region_name=region,
+ )
+
+
@pytest.fixture(scope="module")
def scheduler():
- return boto3.client("scheduler", endpoint_url=ENDPOINT,
- aws_access_key_id="test", aws_secret_access_key="test",
- region_name=REGION)
+ return _scheduler_client(REGION)
@pytest.fixture(scope="module")
@@ -60,6 +68,105 @@ def test_scheduler_create_get_delete_group(scheduler):
assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
+def test_scheduler_groups_schedules_and_tags_are_region_scoped(scheduler):
+ west = _scheduler_client("us-west-2")
+ group = f"region-group-{_uid()}"
+ name = f"region-schedule-{_uid()}"
+
+ east_group_arn = scheduler.create_schedule_group(Name=group)["ScheduleGroupArn"]
+ west_group_arn = west.create_schedule_group(Name=group)["ScheduleGroupArn"]
+ assert east_group_arn == f"arn:aws:scheduler:us-east-1:000000000000:schedule-group/{group}"
+ assert west_group_arn == f"arn:aws:scheduler:us-west-2:000000000000:schedule-group/{group}"
+ assert [item["Arn"] for item in scheduler.list_schedule_groups(NamePrefix=group)["ScheduleGroups"]] == [
+ east_group_arn
+ ]
+ assert [item["Arn"] for item in west.list_schedule_groups(NamePrefix=group)["ScheduleGroups"]] == [
+ west_group_arn
+ ]
+
+ east_schedule_arn = scheduler.create_schedule(
+ Name=name,
+ GroupName=group,
+ ScheduleExpression="rate(1 hour)",
+ FlexibleTimeWindow={"Mode": "OFF"},
+ Target={
+ "Arn": "arn:aws:lambda:us-east-1:000000000000:function:east-target",
+ "RoleArn": "arn:aws:iam::000000000000:role/test",
+ },
+ )["ScheduleArn"]
+ west_schedule_arn = west.create_schedule(
+ Name=name,
+ GroupName=group,
+ ScheduleExpression="rate(2 hours)",
+ FlexibleTimeWindow={"Mode": "OFF"},
+ Target={
+ "Arn": "arn:aws:lambda:us-west-2:000000000000:function:west-target",
+ "RoleArn": "arn:aws:iam::000000000000:role/test",
+ },
+ )["ScheduleArn"]
+ assert east_schedule_arn == f"arn:aws:scheduler:us-east-1:000000000000:schedule/{group}/{name}"
+ assert west_schedule_arn == f"arn:aws:scheduler:us-west-2:000000000000:schedule/{group}/{name}"
+
+ assert scheduler.get_schedule(Name=name, GroupName=group)["ScheduleExpression"] == "rate(1 hour)"
+ assert west.get_schedule(Name=name, GroupName=group)["ScheduleExpression"] == "rate(2 hours)"
+ assert [item["Arn"] for item in scheduler.list_schedules(GroupName=group)["Schedules"]] == [
+ east_schedule_arn
+ ]
+ assert [item["Arn"] for item in west.list_schedules(GroupName=group)["Schedules"]] == [
+ west_schedule_arn
+ ]
+
+ scheduler.tag_resource(ResourceArn=east_schedule_arn, Tags=[{"Key": "region", "Value": "east"}])
+ west.tag_resource(ResourceArn=west_schedule_arn, Tags=[{"Key": "region", "Value": "west"}])
+ assert scheduler.list_tags_for_resource(ResourceArn=east_schedule_arn)["Tags"] == [
+ {"Key": "region", "Value": "east"}
+ ]
+ assert west.list_tags_for_resource(ResourceArn=west_schedule_arn)["Tags"] == [
+ {"Key": "region", "Value": "west"}
+ ]
+
+ scheduler.update_schedule(
+ Name=name,
+ GroupName=group,
+ ScheduleExpression="rate(3 hours)",
+ FlexibleTimeWindow={"Mode": "OFF"},
+ Target={
+ "Arn": "arn:aws:lambda:us-east-1:000000000000:function:east-target-updated",
+ "RoleArn": "arn:aws:iam::000000000000:role/test",
+ },
+ )
+ assert scheduler.get_schedule(Name=name, GroupName=group)["ScheduleExpression"] == "rate(3 hours)"
+ assert west.get_schedule(Name=name, GroupName=group)["ScheduleExpression"] == "rate(2 hours)"
+
+ scheduler.delete_schedule(Name=name, GroupName=group)
+ with pytest.raises(ClientError) as exc:
+ scheduler.get_schedule(Name=name, GroupName=group)
+ assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
+ with pytest.raises(ClientError) as exc:
+ scheduler.update_schedule(
+ Name=name,
+ GroupName=group,
+ ScheduleExpression="rate(4 hours)",
+ FlexibleTimeWindow={"Mode": "OFF"},
+ Target={
+ "Arn": "arn:aws:lambda:us-east-1:000000000000:function:missing",
+ "RoleArn": "arn:aws:iam::000000000000:role/test",
+ },
+ )
+ assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
+ with pytest.raises(ClientError) as exc:
+ scheduler.delete_schedule(Name=name, GroupName=group)
+ assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
+
+ scheduler.delete_schedule_group(Name=group)
+ assert west.get_schedule_group(Name=group)["Arn"] == west_group_arn
+ assert west.get_schedule(Name=name, GroupName=group)["Arn"] == west_schedule_arn
+ assert west.list_tags_for_resource(ResourceArn=west_schedule_arn)["Tags"] == [
+ {"Key": "region", "Value": "west"}
+ ]
+ west.delete_schedule_group(Name=group)
+
+
def test_scheduler_create_duplicate_group(scheduler):
name = f"dup-group-{_uid()}"
scheduler.create_schedule_group(Name=name)
diff --git a/tests/test_scheduler_arn_adoption.py b/tests/test_scheduler_arn_adoption.py
index f8d2ff3f..9467e0cb 100644
--- a/tests/test_scheduler_arn_adoption.py
+++ b/tests/test_scheduler_arn_adoption.py
@@ -5,7 +5,12 @@ import uuid
import pytest
import ministack.services.scheduler as scheduler
-from ministack.core.responses import set_request_account_id, set_request_region
+from ministack.core.responses import (
+ get_account_id,
+ get_region,
+ set_request_account_id,
+ set_request_region,
+)
def _uid():
@@ -17,8 +22,12 @@ def _reset_scheduler():
set_request_account_id("000000000000")
set_request_region("us-east-1")
scheduler.reset()
+ scheduler._schedule_last_fired.clear()
+ scheduler._ticker_thread = None
yield
scheduler.reset()
+ scheduler._schedule_last_fired.clear()
+ scheduler._ticker_thread = None
def _request(method, path, body=None, query=None):
@@ -114,6 +123,95 @@ def test_scheduler_tag_apis_do_not_resolve_same_named_resources_from_other_regio
assert scheduler._tags.get(west_group_arn) is None
+def test_scheduler_ticker_dispatches_in_schedule_region_and_restores_context(monkeypatch):
+ from ministack.services import eventbridge
+
+ account_id = "000000000000"
+ name = f"ticker-{_uid()}"
+ key = f"default/{name}"
+ expected = {}
+ for region in ("us-east-1", "us-west-2"):
+ schedule_arn = f"arn:aws:scheduler:{region}:{account_id}:schedule/{key}"
+ target_arn = f"arn:aws:lambda:{region}:{account_id}:function:ticker-target"
+ expected[region] = (schedule_arn, target_arn)
+ scheduler._schedules.set_scoped(
+ account_id,
+ region,
+ key,
+ {
+ "Arn": schedule_arn,
+ "Name": name,
+ "GroupName": "default",
+ "ScheduleExpression": "at(1970-01-01T00:00:01)",
+ "Target": {"Arn": target_arn},
+ "State": "ENABLED",
+ "ActionAfterCompletion": "NONE",
+ "CreationDate": 1,
+ },
+ )
+ set_request_account_id("111111111111")
+ set_request_region("eu-west-1")
+
+ calls = []
+
+ def _capture_target(target, event, schedule):
+ calls.append((get_account_id(), get_region(), target, event, schedule))
+
+ monkeypatch.setattr(eventbridge, "_invoke_target", _capture_target)
+ monkeypatch.setattr(scheduler.time, "time", lambda: 2.0)
+
+ scheduler._tick_schedules()
+
+ assert len(calls) == 2
+ calls_by_region = {call[1]: call for call in calls}
+ assert set(calls_by_region) == set(expected)
+ for region, (schedule_arn, target_arn) in expected.items():
+ dispatch_account, dispatch_region, target, event, schedule = calls_by_region[region]
+ assert (dispatch_account, dispatch_region) == (account_id, region)
+ assert target["Arn"] == target_arn
+ assert event["Account"] == account_id
+ assert event["Region"] == region
+ assert event["Resources"] == [schedule_arn]
+ assert schedule["Arn"] == schedule_arn
+ assert (get_account_id(), get_region()) == ("111111111111", "eu-west-1")
+
+
+def test_scheduler_start_scheduler_starts_daemon_once(monkeypatch):
+ created_threads = []
+
+ class _FakeThread:
+ def __init__(self, *, target, daemon, name):
+ self.target = target
+ self.daemon = daemon
+ self.name = name
+ self.started = False
+ created_threads.append(self)
+
+ def start(self):
+ self.started = True
+
+ def is_alive(self):
+ return self.started
+
+ monkeypatch.setattr(scheduler.threading, "Thread", _FakeThread)
+
+ scheduler.start_scheduler()
+ scheduler.start_scheduler()
+
+ assert len(created_threads) == 1
+ thread = created_threads[0]
+ assert thread.target is scheduler._ticker_loop
+ assert thread.daemon is True
+ assert thread.name == "evb-scheduler-ticker"
+ assert thread.started is True
+
+
+def test_scheduler_unknown_route_returns_validation_error():
+ response = _request("GET", "/unknown")
+
+ _assert_error(response, 400, "ValidationException")
+
+
@pytest.mark.parametrize(
"bad_arn, expected_status, expected_code",
[