Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

Fix persist (#705)

Nahuel Nucera8d46d813f668
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7773a80b..39dcd0de 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -24,6 +24,23 @@ ministack/
└── CHANGELOG.md
```
+## FOR NEW SERVICES — Open an Issue First
+
+> **This section applies only when you are adding a brand-new AWS service** (a new file under `ministack/services/`).
+
+**Before writing any code for a new service, open a GitHub issue.** Use the `enhancement` label and describe:
+
+1. **Which AWS service** (full name + the API namespace, e.g. `inspector2`, `appconfig`, `qbusiness`).
+2. **Which operations** you actually need — not the full API surface. MiniStack favors the operations real users hit (CRUD + the handful of list/get calls SDKs and Terraform providers rely on) over wire-format completeness for every action.
+3. **A real use case** — what tool, framework, or workflow drove the need (Terraform module, CDK construct, app code, integration test, etc.). "I want full parity" is not a use case.
+4. **Scope boundaries** — what's explicitly out of scope for the first PR (e.g. "no async job lifecycle, no findings persistence, no scheduled scans" for Inspector v2). It's fine to ship a partial service; it's not fine to ship a 50-operation stub where 45 return empty lists.
+
+A maintainer will confirm the scope, flag overlap with existing work, and point you at the right protocol (JSON vs XML/Query vs REST) before you write code. This saves you from large PRs that get rejected for scope drift, AWS-parity gaps, or duplicating in-flight work.
+
+**PRs that add a new service without a corresponding scoped issue will be closed and the contributor asked to open one.** This rule is specific to new services — it's the only way we keep MiniStack's AWS parity bar high while staying a one-file-per-service codebase.
+
+---
+
## Adding a New Service
Every service follows the same 4-step pattern:
diff --git a/ministack/app.py b/ministack/app.py
index 154d1f57..dbcf4d1e 100644
--- a/ministack/app.py
+++ b/ministack/app.py
@@ -1700,12 +1700,7 @@ async def _handle_lifespan(scope, receive, send):
elif message["type"] == "lifespan.shutdown":
logger.info("MiniStack shutting down...")
if PERSIST_STATE:
- # Only save state for modules that were actually loaded
- save_dict = {}
- for key, mod_name in _state_map.items():
- if mod_name in _loaded_modules:
- save_dict[key] = _loaded_modules[mod_name].get_state
- save_all(save_dict)
+ save_all(_build_persistence_save_dict())
try:
from ministack.services import transfer
@@ -1749,6 +1744,27 @@ def _stop_docker_containers():
pass
+def _build_persistence_save_dict():
+ """Build the {state_key: get_state} mapping that `save_all` consumes
+ at shutdown. Primary source is `_loaded_modules`, populated by
+ `_get_module()` on every routed request. Falls back to `sys.modules`
+ so modules reached only via sibling imports from other services
+ (e.g. `appsync` -> `appsync_events`, `apigateway` -> `apigateway_v1`,
+ `lambda` -> `cloudwatch_logs` for auto-created log groups, S3
+ notifications -> `eventbridge`) are still persisted. Without this
+ fallback, state created exclusively through cross-service code paths
+ is silently dropped at shutdown (#704 and class)."""
+ save_dict = {}
+ for key, mod_name in _state_map.items():
+ mod = _loaded_modules.get(mod_name)
+ if mod is None:
+ mod = sys.modules.get(f"ministack.services.{mod_name}")
+ if mod is None or not hasattr(mod, "get_state"):
+ continue
+ save_dict[key] = mod.get_state
+ return save_dict
+
+
def _load_persisted_state():
"""Load persisted state for services that support it."""
for svc_key in ("apigateway", "apigateway_v1", "servicediscovery"):
@@ -1759,18 +1775,27 @@ def _load_persisted_state():
# Eagerly import persisted services whose restore path depends on
# a module-level `load_state()` side-effect, but which would not
- # otherwise be imported during startup. These are NOT covered by
- # the explicit central-restore loop above (no
- # `load_persisted_state` method), and the lazy router will not
- # pull them in early enough — for example, `ses_v2` is reached
- # via the `/v2/email/*` path-prefix shortcut and `pipes` via
- # CloudFormation, neither of which fires at lifespan startup.
- # Importing here triggers the restore (and, for `pipes`, also
- # restarts the background poller for any RUNNING pipe). Keep this
- # list narrow — every entry costs a cold-start import. Enforced
- # by `tests/test_persistence_symmetry.py::test_state_map_
- # services_without_endpoint_are_eagerly_imported`.
- for svc_key in ("pipes", "ses_v2"):
+ # otherwise be imported during startup. The lazy router does not
+ # pull them in early enough in any of these cases:
+ # - `ses_v2` is reached via the `/v2/email/*` path-prefix shortcut.
+ # - `pipes` is created only via CloudFormation provisioners.
+ # - `appsync_events` is routable (SERVICE_REGISTRY has
+ # "appsync-events") but real traffic arrives under the
+ # `appsync` credential scope at `/v2/apis`, so the
+ # `appsync-events` lazy handler never fires; the module is
+ # reached only via a sibling import from `appsync.py`, which
+ # bypasses `_get_module` and leaves it out of
+ # `_loaded_modules` → shutdown skips persistence (#704).
+ # - `apigateway_v1` is restored above only when a state file
+ # already exists; on first-ever boot the conditional skips
+ # it, the module is reached only via `apigateway.py`'s
+ # sibling import (line 237), and the first save is silently
+ # dropped. Same bug class as #704.
+ # Importing here triggers the module-level restore (and, for
+ # `pipes`, also restarts the background poller for any RUNNING
+ # pipe). Keep this list narrow — every entry costs a cold-start
+ # import.
+ for svc_key in ("pipes", "ses_v2", "appsync_events", "apigateway_v1"):
_get_module(svc_key)
@@ -1921,18 +1946,28 @@ def _reset_all_state():
# still need reset() — REST API v1 (served via the apigateway module),
# SES v2 (served via the ses module), and EventBridge Pipes (CFN-only
# provisioner with a background poller thread that reset() must stop).
+ # Kept for documentation / safety even though the `sys.modules` fallback
+ # below catches every imported module regardless.
_extra_reset_modules = ("apigateway_v1", "ses_v2", "pipes")
module_names = {cfg["module"] for cfg in SERVICE_REGISTRY.values()}
module_names.update(_extra_reset_modules)
for mod_name in module_names:
- if mod_name in _loaded_modules:
- mod = _loaded_modules[mod_name]
- try:
- mod.reset()
- except Exception as e:
- logger.warning("reset() failed for %s: %s", mod_name, e)
+ # Same class fix as the shutdown save loop: a module reached only via
+ # sibling import from another service (e.g. `appsync` -> `appsync_events`,
+ # `apigateway` -> `apigateway_v1`, `lambda` -> `cloudwatch_logs`) is
+ # imported into `sys.modules` but never registered in `_loaded_modules`.
+ # Without the `sys.modules` fallback, those modules silently skip reset
+ # — leaving state across `/_ministack/reset` calls and breaking test
+ # isolation.
+ mod = _loaded_modules.get(mod_name) or sys.modules.get(f"ministack.services.{mod_name}")
+ if mod is None or not hasattr(mod, "reset"):
+ continue
+ try:
+ mod.reset()
+ except Exception as e:
+ logger.warning("reset() failed for %s: %s", mod_name, e)
S3_DATA_DIR = os.environ.get("S3_DATA_DIR", "/tmp/ministack-data/s3")
S3_PERSIST = os.environ.get("S3_PERSIST", "0") == "1"
diff --git a/ministack/services/rds.py b/ministack/services/rds.py
index d47accfb..cfc8562d 100644
--- a/ministack/services/rds.py
+++ b/ministack/services/rds.py
@@ -163,7 +163,23 @@ def _start_rds_container_for_instance(db_id, instance):
master_pass = instance.get("_MasterUserPassword", "password")
db_name = instance.get("DBName") or "mydb"
endpoint = instance.get("Endpoint") or {}
- host_port = endpoint.get("Port") or _next_port()
+ # Host port must come from `_HostPort` (stored at create time), NOT from
+ # `Endpoint.Port` — the latter is overwritten to `container_port` (e.g.
+ # 5432 for postgres) to match real AWS, so reading it here would try to
+ # bind 5432 on the host and collide on every respawn (#692 follow-up).
+ # Legacy instances persisted before `_HostPort` was stored fall back to a
+ # fresh free port from `_next_port()`.
+ host_port = instance.get("_HostPort") or _next_port()
+ # If the stored host port was claimed by something else between
+ # restarts (another ministack, another db instance, a user app),
+ # docker bind would fail with "port is already allocated". Fall
+ # back to a fresh free port and persist it so subsequent restarts
+ # converge on a stable mapping again.
+ if not _is_host_port_free(host_port):
+ logger.info("RDS: persisted host port %d for %s is in use; "
+ "allocating fresh free port", host_port, db_id)
+ host_port = _next_port()
+ instance["_HostPort"] = host_port
image, env_vars, container_port, data_path = _docker_image_for_engine(
engine, engine_version, master_user, master_pass, db_name,
@@ -175,15 +191,27 @@ def _start_rds_container_for_instance(db_id, instance):
container_name = f"ministack-rds-{db_id}"
try:
existing = docker_client.containers.get(container_name)
+ # `force=True` stops AND removes in one shot, including
+ # half-spawned "Created" containers that didn't fully start
+ # — those still hold port mappings and would collide with
+ # the next `containers.run` (#692 follow-up: doodaz saw
+ # a `Created` container blocking the bind).
try:
- existing.stop(timeout=2)
- except Exception:
- pass
- try:
- existing.remove(v=False)
+ existing.remove(force=True, v=False)
except Exception as e:
logger.warning("RDS: failed to remove stale container %s: %s",
container_name, e)
+ # Verify the name is actually free now; if removal silently
+ # failed, abort respawn rather than crash inside `containers.run`
+ # with a confusing name-conflict error.
+ try:
+ docker_client.containers.get(container_name)
+ logger.warning("RDS: stale container %s still present after "
+ "force-remove — aborting respawn", container_name)
+ instance["DBInstanceStatus"] = "failed"
+ return
+ except Exception:
+ pass # Good — name is gone.
except Exception:
pass # No existing container with that name — fine
@@ -427,8 +455,40 @@ def _refresh_cluster_status(cluster_id):
_port_lock = threading.Lock()
+def _is_host_port_free(port: int) -> bool:
+ """Probe that no other process holds host TCP `port`. Best-effort
+ pre-flight check so respawn can pick a different port instead of
+ failing at `docker run` with `port is already allocated` (#692
+ follow-up). There is a small TOCTOU window between probe and
+ `containers.run`, but it closes the common case of stale or
+ user-process bindings."""
+ import socket
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ try:
+ s.bind(("0.0.0.0", port))
+ return True
+ except OSError:
+ return False
+ finally:
+ s.close()
+
+
def _next_port():
+ """Return the next free host port for an RDS container. Increments
+ the persisted counter, but skips ports that are already bound on the
+ host (e.g. by another ministack instance or the user's own services).
+ Caps probing to avoid infinite loops if the entire upper range is
+ saturated — in that pathological case the caller will get a port and
+ likely fail at `docker run`, but we won't spin forever."""
with _port_lock:
+ for _ in range(200):
+ port = _port_counter[0]
+ _port_counter[0] += 1
+ if _is_host_port_free(port):
+ return port
+ # Saturated — return the next counter value and let docker surface
+ # whatever it surfaces. Better than a silent hang.
port = _port_counter[0]
_port_counter[0] += 1
return port
@@ -637,7 +697,8 @@ def _create_db_instance(p):
)
if image:
try:
- host_port = _next_port()
+ host_port = instance.get("_HostPort") or _next_port()
+ instance["_HostPort"] = host_port
endpoint_port = host_port
container_kwargs = dict(
image=image, detach=True,
diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index 01d64f03..b0098aaa 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -121,6 +121,77 @@ def test_state_map_services_without_endpoint_are_eagerly_imported():
)
+def test_save_dict_includes_sibling_imported_modules():
+ """Regression for #704. `appsync_events` is reached only via sibling
+ import from `appsync.py` for REST traffic — the routed handler for
+ `appsync-events` never fires (Event API traffic arrives under the
+ `appsync` credential scope). Same shape: `apigateway_v1` is reached
+ via sibling import from `apigateway.py`. If the shutdown save loop
+ only consults `_loaded_modules` (populated by `_get_module`), those
+ sibling-imported modules are silently skipped and their state is
+ dropped. The fallback through `sys.modules` is the fix."""
+ import sys as _sys
+
+ from ministack.app import _build_persistence_save_dict, _loaded_modules
+
+ # Force-import appsync_events the way appsync.py does it — a plain
+ # sibling import that bypasses `_get_module` and therefore does NOT
+ # populate `_loaded_modules`.
+ import ministack.services.appsync_events # noqa: F401
+
+ # Simulate the bug condition: module is in sys.modules but absent
+ # from _loaded_modules.
+ saved = _loaded_modules.pop("appsync_events", None)
+ try:
+ assert "ministack.services.appsync_events" in _sys.modules, (
+ "test premise broken — module isn't in sys.modules"
+ )
+ assert "appsync_events" not in _loaded_modules, (
+ "test premise broken — module is still in _loaded_modules"
+ )
+
+ save_dict = _build_persistence_save_dict()
+
+ assert "appsync_events" in save_dict, (
+ "shutdown save loop dropped appsync_events even though the "
+ "module was imported via a sibling import. The sys.modules "
+ "fallback in `_build_persistence_save_dict` is missing or broken."
+ )
+ # The value must be the bound get_state method, not the result.
+ assert callable(save_dict["appsync_events"]), (
+ "save_dict should map to a callable (get_state method ref) — "
+ "save_all invokes it. Got %r" % (save_dict["appsync_events"],)
+ )
+ finally:
+ if saved is not None:
+ _loaded_modules["appsync_events"] = saved
+
+
+def test_save_dict_skips_modules_never_imported():
+ """The sys.modules fallback must NOT save state for modules that
+ were never imported at all — there's no state to capture and any
+ `get_state()` call on a non-imported module would attribute-error.
+ Defensive guard: ensure the fallback path's `hasattr` check works."""
+ import sys as _sys
+
+ from ministack.app import _build_persistence_save_dict, _loaded_modules, _state_map
+
+ # Pick any persisted module and ensure it's truly absent from both
+ # `_loaded_modules` and `sys.modules`. `cur` is an obscure one that
+ # most test sessions won't have touched.
+ target = "ecs_metadata" # not in _state_map → guaranteed absent from save_dict
+ assert target not in {v for v in _state_map.values()}, (
+ "test premise broken — pick a module not in _state_map"
+ )
+
+ # Even after the fallback path runs, ecs_metadata must not appear.
+ save_dict = _build_persistence_save_dict()
+ assert "ecs_metadata" not in save_dict, (
+ "save_dict picked up a module that isn't even in _state_map — "
+ "the loop's key-membership check is broken."
+ )
+
+
# ── Functional round-trip tests ────────────────────────────────────────
def _round_trip(mod_name, svc_key, populate_fn, observe_fn):
diff --git a/tests/test_rds.py b/tests/test_rds.py
index bcf7d350..f70503a6 100644
--- a/tests/test_rds.py
+++ b/tests/test_rds.py
@@ -1562,14 +1562,18 @@ def test_rds_restore_state_removes_stale_container_before_respawn(monkeypatch):
def reload(self): pass
def stop(self, timeout=2): pass
- def remove(self, v=False):
+ def remove(self, **kwargs):
removed.append(self.name)
stale = FakeContainer(name="ministack-rds-stale-db", container_id="cid-stale")
class FakeContainers:
def get(self, name):
- if name == "ministack-rds-stale-db":
+ # First call returns the stale container; after .remove() is
+ # called and `removed` is populated, subsequent .get()s for
+ # the same name raise "not found" — mirrors real docker after
+ # a successful force-remove.
+ if name == "ministack-rds-stale-db" and "ministack-rds-stale-db" not in removed:
return stale
raise Exception("not found")
@@ -1615,6 +1619,256 @@ def test_rds_restore_state_removes_stale_container_before_respawn(monkeypatch):
m._instances.clear()
+def test_rds_respawn_does_not_bind_engine_port_on_host(monkeypatch):
+ """Real AWS reports `Endpoint.Port` as the engine's standard port
+ (5432 for postgres) regardless of the docker host port mapping.
+ Respawn must NOT read `Endpoint.Port` as the host bind port — doing
+ so makes every restart try to bind 0.0.0.0:5432 and collide.
+ Regression for the bug doodaz reported on #692 after 1.3.48: the
+ 1.3.47 + 1.3.48 fixes covered restore-then-respawn but left this
+ port-reuse bug live."""
+ from ministack.services import rds as m
+
+ runs = []
+
+ class FakeContainer:
+ def __init__(self, name):
+ self.id = "cid-fake"
+ self.name = name
+ self.attrs = {"NetworkSettings": {"Networks": {}}}
+ def reload(self): pass
+ def stop(self, timeout=2): pass
+ def remove(self, v=False): pass
+
+ class FakeContainers:
+ def get(self, name): raise Exception("not found")
+ def run(self, **kwargs):
+ runs.append(kwargs)
+ return FakeContainer(kwargs["name"])
+
+ class FakeDocker:
+ def __init__(self):
+ self.containers = FakeContainers()
+
+ monkeypatch.setattr(m, "_get_docker", lambda: FakeDocker())
+ monkeypatch.setattr(m, "_get_ministack_network", lambda c: None)
+
+ db_id = "port-collision-db"
+ # Simulate state as persisted by a real run: Endpoint.Port has been
+ # overwritten to the container port (5432 for postgres) to match AWS.
+ persisted_state = {
+ "instances": {db_id: {
+ "DBInstanceIdentifier": db_id,
+ "Engine": "postgres",
+ "EngineVersion": "16.3",
+ "MasterUsername": "admin",
+ "_MasterUserPassword": "password123",
+ "DBName": "mydb",
+ "DBInstanceStatus": "available",
+ "Endpoint": {"Address": "localhost", "Port": 5432, "HostedZoneId": "Z"},
+ }},
+ "clusters": {}, "subnet_groups": {}, "param_groups": {},
+ "snapshots": {}, "db_cluster_param_groups": {},
+ "db_cluster_snapshots": {}, "option_groups": {},
+ "global_clusters": {}, "tags": {}, "port_counter": 15500,
+ }
+
+ m._instances.clear()
+ m.restore_state(persisted_state)
+
+ deadline = time.time() + 5
+ while time.time() < deadline and not runs:
+ time.sleep(0.05)
+
+ assert runs, "restore_state did not respawn the Docker container"
+ port_mapping = runs[0]["ports"]
+ # ports == {"5432/tcp": host_port} — host_port must NOT be 5432.
+ assert port_mapping == {"5432/tcp": runs[0]["ports"]["5432/tcp"]}, port_mapping
+ host_port = port_mapping["5432/tcp"]
+ assert host_port != 5432, (
+ f"respawn tried to bind container port 5432 to host port 5432 — "
+ f"will collide with anything else listening on 5432. "
+ f"ports={port_mapping}"
+ )
+ assert host_port >= 15432, (
+ f"respawn host port {host_port} not in MiniStack's allocated range "
+ f"(>=15432) — looks like Endpoint.Port leaked through again."
+ )
+
+ # The instance must now carry _HostPort so subsequent respawns reuse
+ # the same host port instead of allocating a new one each time.
+ restored = m._instances.get(db_id)
+ assert restored.get("_HostPort") == host_port, (
+ "respawn did not persist _HostPort on the instance — next restart "
+ "will pick a different port and break clients with cached connection strings."
+ )
+
+ m._instances.clear()
+
+
+def test_rds_next_port_skips_busy_ports(monkeypatch):
+ """`_next_port` must probe each candidate and skip ports already
+ bound on the host. Without this, a counter-only allocator hands out
+ a port that `docker run` will immediately fail to bind."""
+ from ministack.services import rds as m
+
+ busy_ports = {15432, 15433, 15434}
+ monkeypatch.setattr(m, "_is_host_port_free", lambda p: p not in busy_ports)
+ monkeypatch.setattr(m, "_port_counter", [15432])
+
+ port = m._next_port()
+ assert port == 15435, (
+ f"_next_port returned {port} but ports 15432-15434 were busy — "
+ f"it must skip taken ports, not blindly hand them out."
+ )
+
+
+def test_rds_respawn_falls_back_when_persisted_host_port_taken(monkeypatch):
+ """If `_HostPort` from persisted state is taken by another process
+ on the host, respawn must fall back to a fresh free port instead
+ of trying to bind a port we know is unavailable."""
+ from ministack.services import rds as m
+
+ runs = []
+
+ class FakeContainer:
+ def __init__(self, name):
+ self.id = "cid-fake"; self.name = name
+ self.attrs = {"NetworkSettings": {"Networks": {}}}
+ def reload(self): pass
+ def stop(self, timeout=2): pass
+ def remove(self, **kwargs): pass
+
+ class FakeContainers:
+ def get(self, name): raise Exception("not found")
+ def run(self, **kwargs):
+ runs.append(kwargs)
+ return FakeContainer(kwargs["name"])
+
+ class FakeDocker:
+ def __init__(self): self.containers = FakeContainers()
+
+ monkeypatch.setattr(m, "_get_docker", lambda: FakeDocker())
+ monkeypatch.setattr(m, "_get_ministack_network", lambda c: None)
+ # Persisted port 15500 is "taken"; anything else is free.
+ monkeypatch.setattr(m, "_is_host_port_free", lambda p: p != 15500)
+ monkeypatch.setattr(m, "_port_counter", [15600])
+
+ db_id = "fallback-db"
+ persisted_state = {
+ "instances": {db_id: {
+ "DBInstanceIdentifier": db_id,
+ "Engine": "postgres",
+ "EngineVersion": "16.3",
+ "MasterUsername": "admin",
+ "_MasterUserPassword": "p",
+ "DBName": "mydb",
+ "DBInstanceStatus": "available",
+ "Endpoint": {"Address": "localhost", "Port": 5432, "HostedZoneId": "Z"},
+ "_HostPort": 15500,
+ }},
+ "clusters": {}, "subnet_groups": {}, "param_groups": {},
+ "snapshots": {}, "db_cluster_param_groups": {},
+ "db_cluster_snapshots": {}, "option_groups": {},
+ "global_clusters": {}, "tags": {}, "port_counter": 15600,
+ }
+
+ m._instances.clear()
+ m.restore_state(persisted_state)
+
+ deadline = time.time() + 5
+ while time.time() < deadline and not runs:
+ time.sleep(0.05)
+
+ assert runs, "respawn never happened"
+ host_port = runs[0]["ports"]["5432/tcp"]
+ assert host_port != 15500, (
+ f"respawn used stored _HostPort=15500 despite it being busy — "
+ f"will fail at docker bind. Got host_port={host_port}."
+ )
+ assert m._instances[db_id]["_HostPort"] == host_port, (
+ "fallback port not persisted on instance — next restart will "
+ "try the same busy port again."
+ )
+
+ m._instances.clear()
+
+
+def test_rds_respawn_force_removes_stale_created_container(monkeypatch):
+ """Doodaz observed a half-spawned `Created`-status container with
+ the deterministic name blocking respawn. Plain `.remove()` doesn't
+ handle running/created containers; respawn must use `force=True`."""
+ from ministack.services import rds as m
+
+ remove_calls = []
+
+ class FakeStaleContainer:
+ def remove(self, **kwargs):
+ remove_calls.append(kwargs)
+ if not kwargs.get("force"):
+ raise Exception("cannot remove non-stopped container without force")
+
+ class FakeContainer:
+ def __init__(self, name):
+ self.id = "cid-fake"; self.name = name
+ self.attrs = {"NetworkSettings": {"Networks": {}}}
+ def reload(self): pass
+ def stop(self, timeout=2): pass
+ def remove(self, **kwargs): pass
+
+ name_present = {"yes": True}
+
+ class FakeContainers:
+ def get(self, name):
+ if name_present["yes"]:
+ name_present["yes"] = False # second .get() (verification) returns "not found"
+ return FakeStaleContainer()
+ raise Exception("not found")
+ def run(self, **kwargs):
+ return FakeContainer(kwargs["name"])
+
+ class FakeDocker:
+ def __init__(self): self.containers = FakeContainers()
+
+ monkeypatch.setattr(m, "_get_docker", lambda: FakeDocker())
+ monkeypatch.setattr(m, "_get_ministack_network", lambda c: None)
+ monkeypatch.setattr(m, "_is_host_port_free", lambda p: True)
+ monkeypatch.setattr(m, "_port_counter", [15700])
+
+ db_id = "stale-created-db"
+ persisted_state = {
+ "instances": {db_id: {
+ "DBInstanceIdentifier": db_id,
+ "Engine": "postgres",
+ "EngineVersion": "16.3",
+ "MasterUsername": "admin",
+ "_MasterUserPassword": "p",
+ "DBName": "mydb",
+ "DBInstanceStatus": "available",
+ "Endpoint": {"Address": "localhost", "Port": 5432, "HostedZoneId": "Z"},
+ }},
+ "clusters": {}, "subnet_groups": {}, "param_groups": {},
+ "snapshots": {}, "db_cluster_param_groups": {},
+ "db_cluster_snapshots": {}, "option_groups": {},
+ "global_clusters": {}, "tags": {}, "port_counter": 15700,
+ }
+
+ m._instances.clear()
+ m.restore_state(persisted_state)
+
+ deadline = time.time() + 5
+ while time.time() < deadline and m._instances.get(db_id, {}).get("DBInstanceStatus") not in ("available", "failed"):
+ time.sleep(0.05)
+
+ assert remove_calls, "respawn did not attempt to remove stale container"
+ assert any(c.get("force") for c in remove_calls), (
+ f"respawn called .remove() without force=True (calls={remove_calls}) — "
+ f"will fail on Created/Running containers like the one doodaz hit."
+ )
+
+ m._instances.clear()
+
+
# ========== from test_rds_lambda_network.py ==========
# RDS+Lambda network reachability via DOCKER_NETWORK auto-detect.
import io