Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix: restore persisted state on startup for 5 services (autoscaling/backup/eks/scheduler/pipes) (#491)
Stephan Mielke1fafddabdaec
diff --git a/ministack/app.py b/ministack/app.pyindex 2e8448e9..98c97579 100644--- a/ministack/app.py+++ b/ministack/app.py@@ -193,6 +193,33 @@ SERVICE_HANDLERS = {for service_name, service_config in SERVICE_REGISTRY.items()}+# Maps the on-disk persistence key to the service module name. `save_all`+# (lifespan.shutdown) consumes this. Restore happens at module import time+# in each service via its own `load_state()` call (see e.g. services/sqs.py);+# a small allow-list is also restored centrally by `_load_persisted_state`+# below. Symmetry between save and restore is enforced by+# tests/test_persistence_symmetry.py.+_state_map = {+ "apigateway": "apigateway", "apigateway_v1": "apigateway_v1",+ "sqs": "sqs", "sns": "sns", "ssm": "ssm",+ "secretsmanager": "secretsmanager", "iam": "iam",+ "dynamodb": "dynamodb", "kms": "kms", "eventbridge": "eventbridge",+ "cloudwatch_logs": "cloudwatch_logs", "kinesis": "kinesis",+ "ec2": "ec2", "route53": "route53", "cognito": "cognito",+ "ecr": "ecr", "cloudwatch": "cloudwatch", "s3": "s3",+ "lambda": "lambda_svc", "rds": "rds", "ecs": "ecs",+ "elasticache": "elasticache", "appsync": "appsync",+ "stepfunctions": "stepfunctions", "alb": "alb",+ "glue": "glue", "efs": "efs", "waf": "waf",+ "athena": "athena", "emr": "emr", "cloudfront": "cloudfront",+ "codebuild": "codebuild", "acm": "acm", "firehose": "firehose",+ "ses": "ses", "ses_v2": "ses_v2",+ "servicediscovery": "servicediscovery", "s3files": "s3files",+ "appconfig": "appconfig", "transfer": "transfer",+ "scheduler": "scheduler", "autoscaling": "autoscaling",+ "eks": "eks", "backup": "backup", "pipes": "pipes",+}+SERVICE_NAME_ALIASES = {alias: service_namefor service_name, service_config in SERVICE_REGISTRY.items()@@ -1133,26 +1160,6 @@ async def _handle_lifespan(scope, receive, send):logger.info("MiniStack shutting down...")if PERSIST_STATE:# Only save state for modules that were actually loaded- _state_map = {- "apigateway": "apigateway", "apigateway_v1": "apigateway_v1",- "sqs": "sqs", "sns": "sns", "ssm": "ssm",- "secretsmanager": "secretsmanager", "iam": "iam",- "dynamodb": "dynamodb", "kms": "kms", "eventbridge": "eventbridge",- "cloudwatch_logs": "cloudwatch_logs", "kinesis": "kinesis",- "ec2": "ec2", "route53": "route53", "cognito": "cognito",- "ecr": "ecr", "cloudwatch": "cloudwatch", "s3": "s3",- "lambda": "lambda_svc", "rds": "rds", "ecs": "ecs",- "elasticache": "elasticache", "appsync": "appsync",- "stepfunctions": "stepfunctions", "alb": "alb",- "glue": "glue", "efs": "efs", "waf": "waf",- "athena": "athena", "emr": "emr", "cloudfront": "cloudfront",- "codebuild": "codebuild", "acm": "acm", "firehose": "firehose",- "ses": "ses", "ses_v2": "ses_v2",- "servicediscovery": "servicediscovery", "s3files": "s3files",- "appconfig": "appconfig", "transfer": "transfer",- "scheduler": "scheduler", "autoscaling": "autoscaling",- "eks": "eks", "backup": "backup",- }save_dict = {}for key, mod_name in _state_map.items():if mod_name in _loaded_modules:diff --git a/ministack/services/autoscaling.py b/ministack/services/autoscaling.pyindex d0e3458f..e9f64d59 100644--- a/ministack/services/autoscaling.py+++ b/ministack/services/autoscaling.py@@ -14,12 +14,14 @@ Supports:Tags: CreateOrUpdateTags, DescribeTags, DeleteTags"""+import copyimport loggingimport osimport timefrom collections import defaultdictfrom ministack.core.responses import AccountScopedDict, get_account_id, new_uuid, now_iso, get_region+from ministack.core.persistence import load_statelogger = logging.getLogger("autoscaling")REGION = os.environ.get("MINISTACK_REGION", "us-east-1")@@ -32,9 +34,6 @@ _scheduled_actions = AccountScopedDict()_tags = AccountScopedDict() # asg_name -> [{"Key":..., "Value":...}, ...]-import copy--def get_state():return {"asgs": copy.deepcopy(_asgs),@@ -56,6 +55,14 @@ def restore_state(data):_tags.update(data.get("tags", {}))+try:+ _restored = load_state("autoscaling")+ if _restored:+ restore_state(_restored)+except Exception:+ logger.exception("Failed to restore persisted autoscaling state; continuing fresh")++def reset():_asgs.clear()_launch_configs.clear()diff --git a/ministack/services/backup.py b/ministack/services/backup.pyindex a18901ee..a149c9e0 100644--- a/ministack/services/backup.py+++ b/ministack/services/backup.py@@ -18,6 +18,7 @@ import loggingimport timefrom ministack.core.responses import AccountScopedDict, get_account_id, get_region, new_uuid+from ministack.core.persistence import load_statelogger = logging.getLogger("backup")@@ -39,12 +40,16 @@ def reset():def get_state():- return copy.deepcopy({- "vaults": dict(_vaults),- "plans": dict(_plans),- "selections": dict(_selections),- "jobs": dict(_jobs),- })+ # Preserve AccountScopedDict wrappers; casting to a plain dict drops+ # the per-account scoping and would persist only the current request's+ # tenants. AccountScopedDict has a JSON encoder hook that round-trips+ # the (account, key) tuple correctly.+ return {+ "vaults": copy.deepcopy(_vaults),+ "plans": copy.deepcopy(_plans),+ "selections": copy.deepcopy(_selections),+ "jobs": copy.deepcopy(_jobs),+ }def restore_state(data):@@ -58,6 +63,14 @@ def load_persisted_state(data):restore_state(data)+try:+ _restored = load_state("backup")+ if _restored:+ restore_state(_restored)+except Exception:+ logger.exception("Failed to restore persisted backup state; continuing fresh")++# ---------------------------------------------------------------------------# Helpers# ---------------------------------------------------------------------------diff --git a/ministack/services/eks.py b/ministack/services/eks.pyindex 9beb65ab..fc00848a 100644--- a/ministack/services/eks.py+++ b/ministack/services/eks.py@@ -22,6 +22,7 @@ import threadingimport timefrom ministack.core.responses import AccountScopedDict, apply_image_prefix, get_account_id, json_response, error_response_json, new_uuid, get_region+from ministack.core.persistence import load_statelogger = logging.getLogger("eks")@@ -89,6 +90,14 @@ def restore_state(data):c["_docker_id"] = None+try:+ _restored = load_state("eks")+ if _restored:+ restore_state(_restored)+except Exception:+ logger.exception("Failed to restore persisted eks state; continuing fresh")++# ---------------------------------------------------------------------------# Helpers# ---------------------------------------------------------------------------diff --git a/ministack/services/pipes.py b/ministack/services/pipes.pyindex c9c1879c..baa7b0b7 100644--- a/ministack/services/pipes.py+++ b/ministack/services/pipes.py@@ -15,6 +15,7 @@ import threadingimport timefrom ministack.core.responses import AccountScopedDict, get_account_id, new_uuid, get_region+from ministack.core.persistence import load_statelogger = logging.getLogger("pipes")@@ -33,6 +34,26 @@ def get_state():}+def restore_state(data):+ if data:+ _pipes.update(data.get("pipes", {}))+ _positions.update(data.get("positions", {}))+ # Restored RUNNING pipes need the background poller — register_pipe+ # is the only other place that starts it, and it isn't called on+ # warm-boot. Without this, persisted pipes would silently stop+ # forwarding events until a new pipe is registered.+ if any(p.get("CurrentState") == "RUNNING" for p in _pipes.values()):+ _ensure_poller()+++try:+ _restored = load_state("pipes")+ if _restored:+ restore_state(_restored)+except Exception:+ logger.exception("Failed to restore persisted pipes state; continuing fresh")++def reset():_pipes.clear()_positions.clear()diff --git a/ministack/services/scheduler.py b/ministack/services/scheduler.pyindex 6cdd1ba9..5735f55c 100644--- a/ministack/services/scheduler.py+++ b/ministack/services/scheduler.py@@ -18,6 +18,7 @@ import reimport timefrom ministack.core.responses import AccountScopedDict, get_account_id, get_region+from ministack.core.persistence import load_statelogger = logging.getLogger("scheduler")@@ -39,11 +40,15 @@ def reset():def get_state():- return copy.deepcopy({- "schedules": dict(_schedules),- "schedule_groups": dict(_schedule_groups),- "tags": dict(_tags),- })+ # Preserve AccountScopedDict wrappers; casting to a plain dict drops the+ # per-account scoping and would persist only the current request's+ # tenants. AccountScopedDict has a JSON encoder hook in core/persistence+ # that round-trips the (account, key) tuple correctly.+ return {+ "schedules": copy.deepcopy(_schedules),+ "schedule_groups": copy.deepcopy(_schedule_groups),+ "tags": copy.deepcopy(_tags),+ }def restore_state(data):@@ -52,6 +57,14 @@ def restore_state(data):_tags.update(data.get("tags", {}))+try:+ _restored = load_state("scheduler")+ if _restored:+ restore_state(_restored)+except Exception:+ logger.exception("Failed to restore persisted scheduler state; continuing fresh")++# ---------------------------------------------------------------------------# Helpers# ---------------------------------------------------------------------------diff --git a/tests/test_persistence_symmetry.py b/tests/test_persistence_symmetry.pynew file mode 100644index 00000000..c15377d9--- /dev/null+++ b/tests/test_persistence_symmetry.py@@ -0,0 +1,268 @@+"""+Regression tests for the persistence-symmetry architectural bug.++Background+----------+When PERSIST_STATE=1, every service that participates in `_state_map`+(see `ministack/app.py`) is saved on shutdown via `save_all()`. State is+restored on startup either by a service's own `load_state()` call at+module import time, OR by `_load_persisted_state()` which calls a+`load_persisted_state()` method on the service module.++For five services (autoscaling, backup, eks, scheduler, pipes), the+shutdown path persists the state to disk but no restore path runs at+startup, so the next boot starts with an empty store. `pipes` is+additionally missing from `_state_map`, so its state is never even saved.++These tests assert the round-trip works for every persisted service.+"""+import importlib+from pathlib import Path++import pytest++from ministack.app import _state_map # noqa: E402 (intentional internal import)+from ministack.core import persistence+++# Services that MUST be persistence-round-trippable. Every entry of+# `_state_map` qualifies. The set is materialised here so an addition to+# `_state_map` automatically gets coverage.+ALL_PERSISTED_SERVICES = sorted(_state_map.items())+++def _module(mod_name):+ return importlib.import_module(f"ministack.services.{mod_name}")++[email protected]("svc_key,mod_name", ALL_PERSISTED_SERVICES)+def test_service_has_restore_path(svc_key, mod_name):+ """Every service in `_state_map` must expose a way to restore its own state.++ Either:+ (a) the module calls `load_state()` itself at import time, OR+ (b) the module exposes `load_persisted_state(data)` AND is wired into+ `_load_persisted_state()` in app.py.+ """+ mod = _module(mod_name)+ src = Path(mod.__file__).read_text()++ # (a) self-restore at import: must import load_state AND call it.+ self_restoring = (+ "from ministack.core.persistence import" in src+ and "load_state" in src+ and "load_state(" in src+ )++ # (b) centrally restored: must define load_persisted_state and be in+ # the explicit allow-list in app.py's `_load_persisted_state()`.+ has_central_method = hasattr(mod, "load_persisted_state")+ centrally_restored = has_central_method and svc_key in {+ "apigateway", "apigateway_v1", "servicediscovery",+ }++ assert self_restoring or centrally_restored, (+ f"Service `{svc_key}` (module `{mod_name}`) is in `_state_map` and "+ f"will be saved on shutdown, but has no restore path on startup. "+ f"Either add `load_state()` at module top, or define "+ f"`load_persisted_state(data)` and add it to "+ f"`_load_persisted_state()` in app.py."+ )+++def test_pipes_is_in_state_map():+ """`pipes` defines `get_state()` so it expects to be persisted, but it+ is missing from `_state_map`. Without this, pipe definitions evaporate+ on every restart even before considering restore-path coverage."""+ pipes = _module("pipes")+ assert hasattr(pipes, "get_state"), "pipes module no longer has get_state — update this test"+ assert "pipes" in _state_map, (+ "`pipes` defines get_state() but is missing from `_state_map` in "+ "app.py — its state is never saved on shutdown."+ )+++# ── Functional round-trip tests ────────────────────────────────────────++def _round_trip(mod_name, svc_key, populate_fn, observe_fn):+ """Helper: populate -> save -> reset -> restore -> observe."""+ mod = _module(mod_name)+ mod.reset()+ populate_fn(mod)+ snapshot = mod.get_state()++ # Persist via the same code path as `save_all` would use.+ persistence.save_state(svc_key, snapshot)++ # Wipe in-memory state — this simulates a process restart.+ mod.reset()++ # Restore via the same code path the module would use at import.+ loaded = persistence.load_state(svc_key)+ assert loaded is not None, (+ f"persistence.load_state({svc_key!r}) returned None — state file "+ "was not written by save_state(). Check `_state_map` membership "+ "and `get_state()` correctness."+ )+ if hasattr(mod, "restore_state"):+ mod.restore_state(loaded)+ elif hasattr(mod, "load_persisted_state"):+ mod.load_persisted_state(loaded)+ else:+ pytest.fail(+ f"Module {mod_name} has neither restore_state nor "+ "load_persisted_state — cannot restore."+ )++ # Cleanup state file before observation, so a failure doesn't pollute+ # the next test run.+ import os+ state_file = os.path.join(persistence.STATE_DIR, f"{svc_key}.json")+ if os.path.exists(state_file):+ os.remove(state_file)++ observe_fn(mod)+ mod.reset()++[email protected](autouse=True)+def _enable_persistence(monkeypatch, tmp_path):+ monkeypatch.setattr(persistence, "PERSIST_STATE", True)+ monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))+++def test_autoscaling_round_trip():+ def populate(mod):+ # Drive the state via the module's own dict directly — minimal+ # surface, no SDK needed.+ mod._launch_configs["lc-test"] = {"LaunchConfigurationName": "lc-test"}+ mod._asgs["asg-test"] = {"AutoScalingGroupName": "asg-test", "MinSize": 1}++ def observe(mod):+ assert "lc-test" in mod._launch_configs+ assert "asg-test" in mod._asgs++ _round_trip("autoscaling", "autoscaling", populate, observe)+++def test_backup_round_trip():+ def populate(mod):+ mod._vaults["vault-test"] = {"BackupVaultName": "vault-test"}++ def observe(mod):+ assert "vault-test" in mod._vaults++ _round_trip("backup", "backup", populate, observe)+++def test_eks_round_trip():+ def populate(mod):+ mod._clusters["cluster-test"] = {"name": "cluster-test", "status": "ACTIVE"}++ def observe(mod):+ assert "cluster-test" in mod._clusters++ _round_trip("eks", "eks", populate, observe)+++def test_scheduler_round_trip():+ # Production code keys _schedules by `f"{group}/{name}"` strings (see+ # scheduler.py CreateSchedule etc.), not tuples — even though the+ # pre-existing inline comment on the dict mis-describes the shape. Use+ # the real production key shape so this test catches a regression that+ # broke string-key serialisation.+ def populate(mod):+ mod._schedule_groups["default"] = {"Name": "default"}+ mod._schedules["default/sched-test"] = {"Name": "sched-test"}++ def observe(mod):+ assert "default" in mod._schedule_groups+ assert "default/sched-test" in mod._schedules++ _round_trip("scheduler", "scheduler", populate, observe)+++def test_pipes_round_trip():+ # Use a complete pipe record matching `register_pipe()` shape so the+ # background poller (which the restore path may start) doesn't blow up+ # on KeyError if it iterates this entry. Source/Target are intentionally+ # non-DDB/non-SNS so `_poll_once` skips them quickly.+ pipe_arn = "arn:aws:pipes:us-east-1:000000000000:pipe/pipe-test"++ def populate(mod):+ mod._pipes["pipe-test"] = {+ "Name": "pipe-test",+ "Arn": pipe_arn,+ "RoleArn": "",+ "Source": "arn:aws:sqs:us-east-1:000000000000:irrelevant",+ "Target": "arn:aws:sqs:us-east-1:000000000000:irrelevant",+ "DesiredState": "STOPPED",+ "CurrentState": "STOPPED",+ "StartingPosition": "LATEST",+ "Tags": {},+ "CreationTime": 0,+ }+ mod._positions[pipe_arn] = 0++ def observe(mod):+ assert "pipe-test" in mod._pipes+ assert mod._positions.get(pipe_arn) == 0++ _round_trip("pipes", "pipes", populate, observe)+++def test_pipes_restore_starts_poller_for_running_pipes(monkeypatch):+ """When `restore_state` reloads pipes that are RUNNING, the background+ poller must be (re)started so events keep flowing after warm-boot."""+ mod = _module("pipes")+ mod.reset()+ # Reset the poller flag so this test is independent of execution order.+ monkeypatch.setattr(mod, "_poller_started", False)++ pipe_arn = "arn:aws:pipes:us-east-1:000000000000:pipe/poller-test"+ mod.restore_state({+ "pipes": {+ "poller-test": {+ "Name": "poller-test",+ "Arn": pipe_arn,+ "RoleArn": "",+ "Source": "arn:aws:sqs:us-east-1:000000000000:irrelevant",+ "Target": "arn:aws:sqs:us-east-1:000000000000:irrelevant",+ "DesiredState": "RUNNING",+ "CurrentState": "RUNNING",+ "StartingPosition": "LATEST",+ "Tags": {},+ "CreationTime": 0,+ },+ },+ "positions": {pipe_arn: 0},+ })++ assert mod._poller_started, (+ "restore_state() did not start the pipes poller for a RUNNING pipe — "+ "warm-booted pipes would silently stop forwarding events."+ )+ mod.reset()+++# ── PERSIST_STATE gating ──────────────────────────────────────────────+[email protected]("svc_key", [+ "autoscaling", "backup", "eks", "scheduler", "pipes",+])+def test_load_state_is_noop_when_persist_state_disabled(monkeypatch, svc_key, tmp_path):+ """When PERSIST_STATE=0, load_state() must return None without touching+ disk and without invoking restore_state(). Catches a regression where+ a service module accidentally calls restore_state() unconditionally."""+ monkeypatch.setattr(persistence, "PERSIST_STATE", False)+ # Pre-write a state file that *would* succeed if persistence were on,+ # so we can assert that it is NOT consumed.+ monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))+ bogus_path = tmp_path / f"{svc_key}.json"+ bogus_path.write_text('{"would_have_been_restored": true}')++ result = persistence.load_state(svc_key)+ assert result is None, (+ f"load_state({svc_key!r}) returned non-None even though "+ "PERSIST_STATE is False — restore must be gated."+ )