Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix: restore persisted state on startup for 5 services (autoscaling/backup/eks/scheduler/pipes) (#491)

Stephan Mielke1fafddabdaec
diff --git a/ministack/app.py b/ministack/app.py
index 2e8448e9..98c97579 100644
--- a/ministack/app.py
+++ b/ministack/app.py
@@ -193,6 +193,33 @@ SERVICE_HANDLERS = {
for service_name, service_config in SERVICE_REGISTRY.items()
}
+# Maps the on-disk persistence key to the service module name. `save_all`
+# (lifespan.shutdown) consumes this. Restore happens at module import time
+# in each service via its own `load_state()` call (see e.g. services/sqs.py);
+# a small allow-list is also restored centrally by `_load_persisted_state`
+# below. Symmetry between save and restore is enforced by
+# tests/test_persistence_symmetry.py.
+_state_map = {
+ "apigateway": "apigateway", "apigateway_v1": "apigateway_v1",
+ "sqs": "sqs", "sns": "sns", "ssm": "ssm",
+ "secretsmanager": "secretsmanager", "iam": "iam",
+ "dynamodb": "dynamodb", "kms": "kms", "eventbridge": "eventbridge",
+ "cloudwatch_logs": "cloudwatch_logs", "kinesis": "kinesis",
+ "ec2": "ec2", "route53": "route53", "cognito": "cognito",
+ "ecr": "ecr", "cloudwatch": "cloudwatch", "s3": "s3",
+ "lambda": "lambda_svc", "rds": "rds", "ecs": "ecs",
+ "elasticache": "elasticache", "appsync": "appsync",
+ "stepfunctions": "stepfunctions", "alb": "alb",
+ "glue": "glue", "efs": "efs", "waf": "waf",
+ "athena": "athena", "emr": "emr", "cloudfront": "cloudfront",
+ "codebuild": "codebuild", "acm": "acm", "firehose": "firehose",
+ "ses": "ses", "ses_v2": "ses_v2",
+ "servicediscovery": "servicediscovery", "s3files": "s3files",
+ "appconfig": "appconfig", "transfer": "transfer",
+ "scheduler": "scheduler", "autoscaling": "autoscaling",
+ "eks": "eks", "backup": "backup", "pipes": "pipes",
+}
+
SERVICE_NAME_ALIASES = {
alias: service_name
for service_name, service_config in SERVICE_REGISTRY.items()
@@ -1133,26 +1160,6 @@ async def _handle_lifespan(scope, receive, send):
logger.info("MiniStack shutting down...")
if PERSIST_STATE:
# Only save state for modules that were actually loaded
- _state_map = {
- "apigateway": "apigateway", "apigateway_v1": "apigateway_v1",
- "sqs": "sqs", "sns": "sns", "ssm": "ssm",
- "secretsmanager": "secretsmanager", "iam": "iam",
- "dynamodb": "dynamodb", "kms": "kms", "eventbridge": "eventbridge",
- "cloudwatch_logs": "cloudwatch_logs", "kinesis": "kinesis",
- "ec2": "ec2", "route53": "route53", "cognito": "cognito",
- "ecr": "ecr", "cloudwatch": "cloudwatch", "s3": "s3",
- "lambda": "lambda_svc", "rds": "rds", "ecs": "ecs",
- "elasticache": "elasticache", "appsync": "appsync",
- "stepfunctions": "stepfunctions", "alb": "alb",
- "glue": "glue", "efs": "efs", "waf": "waf",
- "athena": "athena", "emr": "emr", "cloudfront": "cloudfront",
- "codebuild": "codebuild", "acm": "acm", "firehose": "firehose",
- "ses": "ses", "ses_v2": "ses_v2",
- "servicediscovery": "servicediscovery", "s3files": "s3files",
- "appconfig": "appconfig", "transfer": "transfer",
- "scheduler": "scheduler", "autoscaling": "autoscaling",
- "eks": "eks", "backup": "backup",
- }
save_dict = {}
for key, mod_name in _state_map.items():
if mod_name in _loaded_modules:
diff --git a/ministack/services/autoscaling.py b/ministack/services/autoscaling.py
index d0e3458f..e9f64d59 100644
--- a/ministack/services/autoscaling.py
+++ b/ministack/services/autoscaling.py
@@ -14,12 +14,14 @@ Supports:
Tags: CreateOrUpdateTags, DescribeTags, DeleteTags
"""
+import copy
import logging
import os
import time
from collections import defaultdict
from ministack.core.responses import AccountScopedDict, get_account_id, new_uuid, now_iso, get_region
+from ministack.core.persistence import load_state
logger = logging.getLogger("autoscaling")
REGION = os.environ.get("MINISTACK_REGION", "us-east-1")
@@ -32,9 +34,6 @@ _scheduled_actions = AccountScopedDict()
_tags = AccountScopedDict() # asg_name -> [{"Key":..., "Value":...}, ...]
-import copy
-
-
def get_state():
return {
"asgs": copy.deepcopy(_asgs),
@@ -56,6 +55,14 @@ def restore_state(data):
_tags.update(data.get("tags", {}))
+try:
+ _restored = load_state("autoscaling")
+ if _restored:
+ restore_state(_restored)
+except Exception:
+ logger.exception("Failed to restore persisted autoscaling state; continuing fresh")
+
+
def reset():
_asgs.clear()
_launch_configs.clear()
diff --git a/ministack/services/backup.py b/ministack/services/backup.py
index a18901ee..a149c9e0 100644
--- a/ministack/services/backup.py
+++ b/ministack/services/backup.py
@@ -18,6 +18,7 @@ import logging
import time
from ministack.core.responses import AccountScopedDict, get_account_id, get_region, new_uuid
+from ministack.core.persistence import load_state
logger = logging.getLogger("backup")
@@ -39,12 +40,16 @@ def reset():
def get_state():
- return copy.deepcopy({
- "vaults": dict(_vaults),
- "plans": dict(_plans),
- "selections": dict(_selections),
- "jobs": dict(_jobs),
- })
+ # Preserve AccountScopedDict wrappers; casting to a plain dict drops
+ # the per-account scoping and would persist only the current request's
+ # tenants. AccountScopedDict has a JSON encoder hook that round-trips
+ # the (account, key) tuple correctly.
+ return {
+ "vaults": copy.deepcopy(_vaults),
+ "plans": copy.deepcopy(_plans),
+ "selections": copy.deepcopy(_selections),
+ "jobs": copy.deepcopy(_jobs),
+ }
def restore_state(data):
@@ -58,6 +63,14 @@ def load_persisted_state(data):
restore_state(data)
+try:
+ _restored = load_state("backup")
+ if _restored:
+ restore_state(_restored)
+except Exception:
+ logger.exception("Failed to restore persisted backup state; continuing fresh")
+
+
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
diff --git a/ministack/services/eks.py b/ministack/services/eks.py
index 9beb65ab..fc00848a 100644
--- a/ministack/services/eks.py
+++ b/ministack/services/eks.py
@@ -22,6 +22,7 @@ import threading
import time
from ministack.core.responses import AccountScopedDict, apply_image_prefix, get_account_id, json_response, error_response_json, new_uuid, get_region
+from ministack.core.persistence import load_state
logger = logging.getLogger("eks")
@@ -89,6 +90,14 @@ def restore_state(data):
c["_docker_id"] = None
+try:
+ _restored = load_state("eks")
+ if _restored:
+ restore_state(_restored)
+except Exception:
+ logger.exception("Failed to restore persisted eks state; continuing fresh")
+
+
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
diff --git a/ministack/services/pipes.py b/ministack/services/pipes.py
index c9c1879c..baa7b0b7 100644
--- a/ministack/services/pipes.py
+++ b/ministack/services/pipes.py
@@ -15,6 +15,7 @@ import threading
import time
from ministack.core.responses import AccountScopedDict, get_account_id, new_uuid, get_region
+from ministack.core.persistence import load_state
logger = logging.getLogger("pipes")
@@ -33,6 +34,26 @@ def get_state():
}
+def restore_state(data):
+ if data:
+ _pipes.update(data.get("pipes", {}))
+ _positions.update(data.get("positions", {}))
+ # Restored RUNNING pipes need the background poller — register_pipe
+ # is the only other place that starts it, and it isn't called on
+ # warm-boot. Without this, persisted pipes would silently stop
+ # forwarding events until a new pipe is registered.
+ if any(p.get("CurrentState") == "RUNNING" for p in _pipes.values()):
+ _ensure_poller()
+
+
+try:
+ _restored = load_state("pipes")
+ if _restored:
+ restore_state(_restored)
+except Exception:
+ logger.exception("Failed to restore persisted pipes state; continuing fresh")
+
+
def reset():
_pipes.clear()
_positions.clear()
diff --git a/ministack/services/scheduler.py b/ministack/services/scheduler.py
index 6cdd1ba9..5735f55c 100644
--- a/ministack/services/scheduler.py
+++ b/ministack/services/scheduler.py
@@ -18,6 +18,7 @@ import re
import time
from ministack.core.responses import AccountScopedDict, get_account_id, get_region
+from ministack.core.persistence import load_state
logger = logging.getLogger("scheduler")
@@ -39,11 +40,15 @@ def reset():
def get_state():
- return copy.deepcopy({
- "schedules": dict(_schedules),
- "schedule_groups": dict(_schedule_groups),
- "tags": dict(_tags),
- })
+ # Preserve AccountScopedDict wrappers; casting to a plain dict drops the
+ # per-account scoping and would persist only the current request's
+ # tenants. AccountScopedDict has a JSON encoder hook in core/persistence
+ # that round-trips the (account, key) tuple correctly.
+ return {
+ "schedules": copy.deepcopy(_schedules),
+ "schedule_groups": copy.deepcopy(_schedule_groups),
+ "tags": copy.deepcopy(_tags),
+ }
def restore_state(data):
@@ -52,6 +57,14 @@ def restore_state(data):
_tags.update(data.get("tags", {}))
+try:
+ _restored = load_state("scheduler")
+ if _restored:
+ restore_state(_restored)
+except Exception:
+ logger.exception("Failed to restore persisted scheduler state; continuing fresh")
+
+
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
diff --git a/tests/test_persistence_symmetry.py b/tests/test_persistence_symmetry.py
new file mode 100644
index 00000000..c15377d9
--- /dev/null
+++ b/tests/test_persistence_symmetry.py
@@ -0,0 +1,268 @@
+"""
+Regression tests for the persistence-symmetry architectural bug.
+
+Background
+----------
+When PERSIST_STATE=1, every service that participates in `_state_map`
+(see `ministack/app.py`) is saved on shutdown via `save_all()`. State is
+restored on startup either by a service's own `load_state()` call at
+module import time, OR by `_load_persisted_state()` which calls a
+`load_persisted_state()` method on the service module.
+
+For five services (autoscaling, backup, eks, scheduler, pipes), the
+shutdown path persists the state to disk but no restore path runs at
+startup, so the next boot starts with an empty store. `pipes` is
+additionally missing from `_state_map`, so its state is never even saved.
+
+These tests assert the round-trip works for every persisted service.
+"""
+import importlib
+from pathlib import Path
+
+import pytest
+
+from ministack.app import _state_map # noqa: E402 (intentional internal import)
+from ministack.core import persistence
+
+
+# Services that MUST be persistence-round-trippable. Every entry of
+# `_state_map` qualifies. The set is materialised here so an addition to
+# `_state_map` automatically gets coverage.
+ALL_PERSISTED_SERVICES = sorted(_state_map.items())
+
+
+def _module(mod_name):
+ return importlib.import_module(f"ministack.services.{mod_name}")
+
+
[email protected]("svc_key,mod_name", ALL_PERSISTED_SERVICES)
+def test_service_has_restore_path(svc_key, mod_name):
+ """Every service in `_state_map` must expose a way to restore its own state.
+
+ Either:
+ (a) the module calls `load_state()` itself at import time, OR
+ (b) the module exposes `load_persisted_state(data)` AND is wired into
+ `_load_persisted_state()` in app.py.
+ """
+ mod = _module(mod_name)
+ src = Path(mod.__file__).read_text()
+
+ # (a) self-restore at import: must import load_state AND call it.
+ self_restoring = (
+ "from ministack.core.persistence import" in src
+ and "load_state" in src
+ and "load_state(" in src
+ )
+
+ # (b) centrally restored: must define load_persisted_state and be in
+ # the explicit allow-list in app.py's `_load_persisted_state()`.
+ has_central_method = hasattr(mod, "load_persisted_state")
+ centrally_restored = has_central_method and svc_key in {
+ "apigateway", "apigateway_v1", "servicediscovery",
+ }
+
+ assert self_restoring or centrally_restored, (
+ f"Service `{svc_key}` (module `{mod_name}`) is in `_state_map` and "
+ f"will be saved on shutdown, but has no restore path on startup. "
+ f"Either add `load_state()` at module top, or define "
+ f"`load_persisted_state(data)` and add it to "
+ f"`_load_persisted_state()` in app.py."
+ )
+
+
+def test_pipes_is_in_state_map():
+ """`pipes` defines `get_state()` so it expects to be persisted, but it
+ is missing from `_state_map`. Without this, pipe definitions evaporate
+ on every restart even before considering restore-path coverage."""
+ pipes = _module("pipes")
+ assert hasattr(pipes, "get_state"), "pipes module no longer has get_state — update this test"
+ assert "pipes" in _state_map, (
+ "`pipes` defines get_state() but is missing from `_state_map` in "
+ "app.py — its state is never saved on shutdown."
+ )
+
+
+# ── Functional round-trip tests ────────────────────────────────────────
+
+def _round_trip(mod_name, svc_key, populate_fn, observe_fn):
+ """Helper: populate -> save -> reset -> restore -> observe."""
+ mod = _module(mod_name)
+ mod.reset()
+ populate_fn(mod)
+ snapshot = mod.get_state()
+
+ # Persist via the same code path as `save_all` would use.
+ persistence.save_state(svc_key, snapshot)
+
+ # Wipe in-memory state — this simulates a process restart.
+ mod.reset()
+
+ # Restore via the same code path the module would use at import.
+ loaded = persistence.load_state(svc_key)
+ assert loaded is not None, (
+ f"persistence.load_state({svc_key!r}) returned None — state file "
+ "was not written by save_state(). Check `_state_map` membership "
+ "and `get_state()` correctness."
+ )
+ if hasattr(mod, "restore_state"):
+ mod.restore_state(loaded)
+ elif hasattr(mod, "load_persisted_state"):
+ mod.load_persisted_state(loaded)
+ else:
+ pytest.fail(
+ f"Module {mod_name} has neither restore_state nor "
+ "load_persisted_state — cannot restore."
+ )
+
+ # Cleanup state file before observation, so a failure doesn't pollute
+ # the next test run.
+ import os
+ state_file = os.path.join(persistence.STATE_DIR, f"{svc_key}.json")
+ if os.path.exists(state_file):
+ os.remove(state_file)
+
+ observe_fn(mod)
+ mod.reset()
+
+
[email protected](autouse=True)
+def _enable_persistence(monkeypatch, tmp_path):
+ monkeypatch.setattr(persistence, "PERSIST_STATE", True)
+ monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
+
+
+def test_autoscaling_round_trip():
+ def populate(mod):
+ # Drive the state via the module's own dict directly — minimal
+ # surface, no SDK needed.
+ mod._launch_configs["lc-test"] = {"LaunchConfigurationName": "lc-test"}
+ mod._asgs["asg-test"] = {"AutoScalingGroupName": "asg-test", "MinSize": 1}
+
+ def observe(mod):
+ assert "lc-test" in mod._launch_configs
+ assert "asg-test" in mod._asgs
+
+ _round_trip("autoscaling", "autoscaling", populate, observe)
+
+
+def test_backup_round_trip():
+ def populate(mod):
+ mod._vaults["vault-test"] = {"BackupVaultName": "vault-test"}
+
+ def observe(mod):
+ assert "vault-test" in mod._vaults
+
+ _round_trip("backup", "backup", populate, observe)
+
+
+def test_eks_round_trip():
+ def populate(mod):
+ mod._clusters["cluster-test"] = {"name": "cluster-test", "status": "ACTIVE"}
+
+ def observe(mod):
+ assert "cluster-test" in mod._clusters
+
+ _round_trip("eks", "eks", populate, observe)
+
+
+def test_scheduler_round_trip():
+ # Production code keys _schedules by `f"{group}/{name}"` strings (see
+ # scheduler.py CreateSchedule etc.), not tuples — even though the
+ # pre-existing inline comment on the dict mis-describes the shape. Use
+ # the real production key shape so this test catches a regression that
+ # broke string-key serialisation.
+ def populate(mod):
+ mod._schedule_groups["default"] = {"Name": "default"}
+ mod._schedules["default/sched-test"] = {"Name": "sched-test"}
+
+ def observe(mod):
+ assert "default" in mod._schedule_groups
+ assert "default/sched-test" in mod._schedules
+
+ _round_trip("scheduler", "scheduler", populate, observe)
+
+
+def test_pipes_round_trip():
+ # Use a complete pipe record matching `register_pipe()` shape so the
+ # background poller (which the restore path may start) doesn't blow up
+ # on KeyError if it iterates this entry. Source/Target are intentionally
+ # non-DDB/non-SNS so `_poll_once` skips them quickly.
+ pipe_arn = "arn:aws:pipes:us-east-1:000000000000:pipe/pipe-test"
+
+ def populate(mod):
+ mod._pipes["pipe-test"] = {
+ "Name": "pipe-test",
+ "Arn": pipe_arn,
+ "RoleArn": "",
+ "Source": "arn:aws:sqs:us-east-1:000000000000:irrelevant",
+ "Target": "arn:aws:sqs:us-east-1:000000000000:irrelevant",
+ "DesiredState": "STOPPED",
+ "CurrentState": "STOPPED",
+ "StartingPosition": "LATEST",
+ "Tags": {},
+ "CreationTime": 0,
+ }
+ mod._positions[pipe_arn] = 0
+
+ def observe(mod):
+ assert "pipe-test" in mod._pipes
+ assert mod._positions.get(pipe_arn) == 0
+
+ _round_trip("pipes", "pipes", populate, observe)
+
+
+def test_pipes_restore_starts_poller_for_running_pipes(monkeypatch):
+ """When `restore_state` reloads pipes that are RUNNING, the background
+ poller must be (re)started so events keep flowing after warm-boot."""
+ mod = _module("pipes")
+ mod.reset()
+ # Reset the poller flag so this test is independent of execution order.
+ monkeypatch.setattr(mod, "_poller_started", False)
+
+ pipe_arn = "arn:aws:pipes:us-east-1:000000000000:pipe/poller-test"
+ mod.restore_state({
+ "pipes": {
+ "poller-test": {
+ "Name": "poller-test",
+ "Arn": pipe_arn,
+ "RoleArn": "",
+ "Source": "arn:aws:sqs:us-east-1:000000000000:irrelevant",
+ "Target": "arn:aws:sqs:us-east-1:000000000000:irrelevant",
+ "DesiredState": "RUNNING",
+ "CurrentState": "RUNNING",
+ "StartingPosition": "LATEST",
+ "Tags": {},
+ "CreationTime": 0,
+ },
+ },
+ "positions": {pipe_arn: 0},
+ })
+
+ assert mod._poller_started, (
+ "restore_state() did not start the pipes poller for a RUNNING pipe — "
+ "warm-booted pipes would silently stop forwarding events."
+ )
+ mod.reset()
+
+
+# ── PERSIST_STATE gating ──────────────────────────────────────────────
+
[email protected]("svc_key", [
+ "autoscaling", "backup", "eks", "scheduler", "pipes",
+])
+def test_load_state_is_noop_when_persist_state_disabled(monkeypatch, svc_key, tmp_path):
+ """When PERSIST_STATE=0, load_state() must return None without touching
+ disk and without invoking restore_state(). Catches a regression where
+ a service module accidentally calls restore_state() unconditionally."""
+ monkeypatch.setattr(persistence, "PERSIST_STATE", False)
+ # Pre-write a state file that *would* succeed if persistence were on,
+ # so we can assert that it is NOT consumed.
+ monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
+ bogus_path = tmp_path / f"{svc_key}.json"
+ bogus_path.write_text('{"would_have_been_restored": true}')
+
+ result = persistence.load_state(svc_key)
+ assert result is None, (
+ f"load_state({svc_key!r}) returned non-None even though "
+ "PERSIST_STATE is False — restore must be gated."
+ )