Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
Fix persist (#705)
Nahuel Nucera8d46d813f668
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 7773a80b..39dcd0de 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -24,6 +24,23 @@ ministack/└── CHANGELOG.md```+## FOR NEW SERVICES — Open an Issue First++> **This section applies only when you are adding a brand-new AWS service** (a new file under `ministack/services/`).++**Before writing any code for a new service, open a GitHub issue.** Use the `enhancement` label and describe:++1. **Which AWS service** (full name + the API namespace, e.g. `inspector2`, `appconfig`, `qbusiness`).+2. **Which operations** you actually need — not the full API surface. MiniStack favors the operations real users hit (CRUD + the handful of list/get calls SDKs and Terraform providers rely on) over wire-format completeness for every action.+3. **A real use case** — what tool, framework, or workflow drove the need (Terraform module, CDK construct, app code, integration test, etc.). "I want full parity" is not a use case.+4. **Scope boundaries** — what's explicitly out of scope for the first PR (e.g. "no async job lifecycle, no findings persistence, no scheduled scans" for Inspector v2). It's fine to ship a partial service; it's not fine to ship a 50-operation stub where 45 return empty lists.++A maintainer will confirm the scope, flag overlap with existing work, and point you at the right protocol (JSON vs XML/Query vs REST) before you write code. This saves you from large PRs that get rejected for scope drift, AWS-parity gaps, or duplicating in-flight work.++**PRs that add a new service without a corresponding scoped issue will be closed and the contributor asked to open one.** This rule is specific to new services — it's the only way we keep MiniStack's AWS parity bar high while staying a one-file-per-service codebase.++---+## Adding a New ServiceEvery service follows the same 4-step pattern:diff --git a/ministack/app.py b/ministack/app.pyindex 154d1f57..dbcf4d1e 100644--- a/ministack/app.py+++ b/ministack/app.py@@ -1700,12 +1700,7 @@ async def _handle_lifespan(scope, receive, send):elif message["type"] == "lifespan.shutdown":logger.info("MiniStack shutting down...")if PERSIST_STATE:- # Only save state for modules that were actually loaded- save_dict = {}- for key, mod_name in _state_map.items():- if mod_name in _loaded_modules:- save_dict[key] = _loaded_modules[mod_name].get_state- save_all(save_dict)+ save_all(_build_persistence_save_dict())try:from ministack.services import transfer@@ -1749,6 +1744,27 @@ def _stop_docker_containers():pass+def _build_persistence_save_dict():+ """Build the {state_key: get_state} mapping that `save_all` consumes+ at shutdown. Primary source is `_loaded_modules`, populated by+ `_get_module()` on every routed request. Falls back to `sys.modules`+ so modules reached only via sibling imports from other services+ (e.g. `appsync` -> `appsync_events`, `apigateway` -> `apigateway_v1`,+ `lambda` -> `cloudwatch_logs` for auto-created log groups, S3+ notifications -> `eventbridge`) are still persisted. Without this+ fallback, state created exclusively through cross-service code paths+ is silently dropped at shutdown (#704 and class)."""+ save_dict = {}+ for key, mod_name in _state_map.items():+ mod = _loaded_modules.get(mod_name)+ if mod is None:+ mod = sys.modules.get(f"ministack.services.{mod_name}")+ if mod is None or not hasattr(mod, "get_state"):+ continue+ save_dict[key] = mod.get_state+ return save_dict++def _load_persisted_state():"""Load persisted state for services that support it."""for svc_key in ("apigateway", "apigateway_v1", "servicediscovery"):@@ -1759,18 +1775,27 @@ def _load_persisted_state():# Eagerly import persisted services whose restore path depends on# a module-level `load_state()` side-effect, but which would not- # otherwise be imported during startup. These are NOT covered by- # the explicit central-restore loop above (no- # `load_persisted_state` method), and the lazy router will not- # pull them in early enough — for example, `ses_v2` is reached- # via the `/v2/email/*` path-prefix shortcut and `pipes` via- # CloudFormation, neither of which fires at lifespan startup.- # Importing here triggers the restore (and, for `pipes`, also- # restarts the background poller for any RUNNING pipe). Keep this- # list narrow — every entry costs a cold-start import. Enforced- # by `tests/test_persistence_symmetry.py::test_state_map_- # services_without_endpoint_are_eagerly_imported`.- for svc_key in ("pipes", "ses_v2"):+ # otherwise be imported during startup. The lazy router does not+ # pull them in early enough in any of these cases:+ # - `ses_v2` is reached via the `/v2/email/*` path-prefix shortcut.+ # - `pipes` is created only via CloudFormation provisioners.+ # - `appsync_events` is routable (SERVICE_REGISTRY has+ # "appsync-events") but real traffic arrives under the+ # `appsync` credential scope at `/v2/apis`, so the+ # `appsync-events` lazy handler never fires; the module is+ # reached only via a sibling import from `appsync.py`, which+ # bypasses `_get_module` and leaves it out of+ # `_loaded_modules` → shutdown skips persistence (#704).+ # - `apigateway_v1` is restored above only when a state file+ # already exists; on first-ever boot the conditional skips+ # it, the module is reached only via `apigateway.py`'s+ # sibling import (line 237), and the first save is silently+ # dropped. Same bug class as #704.+ # Importing here triggers the module-level restore (and, for+ # `pipes`, also restarts the background poller for any RUNNING+ # pipe). Keep this list narrow — every entry costs a cold-start+ # import.+ for svc_key in ("pipes", "ses_v2", "appsync_events", "apigateway_v1"):_get_module(svc_key)@@ -1921,18 +1946,28 @@ def _reset_all_state():# still need reset() — REST API v1 (served via the apigateway module),# SES v2 (served via the ses module), and EventBridge Pipes (CFN-only# provisioner with a background poller thread that reset() must stop).+ # Kept for documentation / safety even though the `sys.modules` fallback+ # below catches every imported module regardless._extra_reset_modules = ("apigateway_v1", "ses_v2", "pipes")module_names = {cfg["module"] for cfg in SERVICE_REGISTRY.values()}module_names.update(_extra_reset_modules)for mod_name in module_names:- if mod_name in _loaded_modules:- mod = _loaded_modules[mod_name]- try:- mod.reset()- except Exception as e:- logger.warning("reset() failed for %s: %s", mod_name, e)+ # Same class fix as the shutdown save loop: a module reached only via+ # sibling import from another service (e.g. `appsync` -> `appsync_events`,+ # `apigateway` -> `apigateway_v1`, `lambda` -> `cloudwatch_logs`) is+ # imported into `sys.modules` but never registered in `_loaded_modules`.+ # Without the `sys.modules` fallback, those modules silently skip reset+ # — leaving state across `/_ministack/reset` calls and breaking test+ # isolation.+ mod = _loaded_modules.get(mod_name) or sys.modules.get(f"ministack.services.{mod_name}")+ if mod is None or not hasattr(mod, "reset"):+ continue+ try:+ mod.reset()+ except Exception as e:+ logger.warning("reset() failed for %s: %s", mod_name, e)S3_DATA_DIR = os.environ.get("S3_DATA_DIR", "/tmp/ministack-data/s3")S3_PERSIST = os.environ.get("S3_PERSIST", "0") == "1"diff --git a/ministack/services/rds.py b/ministack/services/rds.pyindex d47accfb..cfc8562d 100644--- a/ministack/services/rds.py+++ b/ministack/services/rds.py@@ -163,7 +163,23 @@ def _start_rds_container_for_instance(db_id, instance):master_pass = instance.get("_MasterUserPassword", "password")db_name = instance.get("DBName") or "mydb"endpoint = instance.get("Endpoint") or {}- host_port = endpoint.get("Port") or _next_port()+ # Host port must come from `_HostPort` (stored at create time), NOT from+ # `Endpoint.Port` — the latter is overwritten to `container_port` (e.g.+ # 5432 for postgres) to match real AWS, so reading it here would try to+ # bind 5432 on the host and collide on every respawn (#692 follow-up).+ # Legacy instances persisted before `_HostPort` was stored fall back to a+ # fresh free port from `_next_port()`.+ host_port = instance.get("_HostPort") or _next_port()+ # If the stored host port was claimed by something else between+ # restarts (another ministack, another db instance, a user app),+ # docker bind would fail with "port is already allocated". Fall+ # back to a fresh free port and persist it so subsequent restarts+ # converge on a stable mapping again.+ if not _is_host_port_free(host_port):+ logger.info("RDS: persisted host port %d for %s is in use; "+ "allocating fresh free port", host_port, db_id)+ host_port = _next_port()+ instance["_HostPort"] = host_portimage, env_vars, container_port, data_path = _docker_image_for_engine(engine, engine_version, master_user, master_pass, db_name,@@ -175,15 +191,27 @@ def _start_rds_container_for_instance(db_id, instance):container_name = f"ministack-rds-{db_id}"try:existing = docker_client.containers.get(container_name)+ # `force=True` stops AND removes in one shot, including+ # half-spawned "Created" containers that didn't fully start+ # — those still hold port mappings and would collide with+ # the next `containers.run` (#692 follow-up: doodaz saw+ # a `Created` container blocking the bind).try:- existing.stop(timeout=2)- except Exception:- pass- try:- existing.remove(v=False)+ existing.remove(force=True, v=False)except Exception as e:logger.warning("RDS: failed to remove stale container %s: %s",container_name, e)+ # Verify the name is actually free now; if removal silently+ # failed, abort respawn rather than crash inside `containers.run`+ # with a confusing name-conflict error.+ try:+ docker_client.containers.get(container_name)+ logger.warning("RDS: stale container %s still present after "+ "force-remove — aborting respawn", container_name)+ instance["DBInstanceStatus"] = "failed"+ return+ except Exception:+ pass # Good — name is gone.except Exception:pass # No existing container with that name — fine@@ -427,8 +455,40 @@ def _refresh_cluster_status(cluster_id):_port_lock = threading.Lock()+def _is_host_port_free(port: int) -> bool:+ """Probe that no other process holds host TCP `port`. Best-effort+ pre-flight check so respawn can pick a different port instead of+ failing at `docker run` with `port is already allocated` (#692+ follow-up). There is a small TOCTOU window between probe and+ `containers.run`, but it closes the common case of stale or+ user-process bindings."""+ import socket+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)+ s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)+ try:+ s.bind(("0.0.0.0", port))+ return True+ except OSError:+ return False+ finally:+ s.close()++def _next_port():+ """Return the next free host port for an RDS container. Increments+ the persisted counter, but skips ports that are already bound on the+ host (e.g. by another ministack instance or the user's own services).+ Caps probing to avoid infinite loops if the entire upper range is+ saturated — in that pathological case the caller will get a port and+ likely fail at `docker run`, but we won't spin forever."""with _port_lock:+ for _ in range(200):+ port = _port_counter[0]+ _port_counter[0] += 1+ if _is_host_port_free(port):+ return port+ # Saturated — return the next counter value and let docker surface+ # whatever it surfaces. Better than a silent hang.port = _port_counter[0]_port_counter[0] += 1return port@@ -637,7 +697,8 @@ def _create_db_instance(p):)if image:try:- host_port = _next_port()+ host_port = instance.get("_HostPort") or _next_port()+ instance["_HostPort"] = host_portendpoint_port = host_portcontainer_kwargs = dict(image=image, detach=True,diff --git a/tests/test_persistence.py b/tests/test_persistence.pyindex 01d64f03..b0098aaa 100644--- a/tests/test_persistence.py+++ b/tests/test_persistence.py@@ -121,6 +121,77 @@ def test_state_map_services_without_endpoint_are_eagerly_imported():)+def test_save_dict_includes_sibling_imported_modules():+ """Regression for #704. `appsync_events` is reached only via sibling+ import from `appsync.py` for REST traffic — the routed handler for+ `appsync-events` never fires (Event API traffic arrives under the+ `appsync` credential scope). Same shape: `apigateway_v1` is reached+ via sibling import from `apigateway.py`. If the shutdown save loop+ only consults `_loaded_modules` (populated by `_get_module`), those+ sibling-imported modules are silently skipped and their state is+ dropped. The fallback through `sys.modules` is the fix."""+ import sys as _sys++ from ministack.app import _build_persistence_save_dict, _loaded_modules++ # Force-import appsync_events the way appsync.py does it — a plain+ # sibling import that bypasses `_get_module` and therefore does NOT+ # populate `_loaded_modules`.+ import ministack.services.appsync_events # noqa: F401++ # Simulate the bug condition: module is in sys.modules but absent+ # from _loaded_modules.+ saved = _loaded_modules.pop("appsync_events", None)+ try:+ assert "ministack.services.appsync_events" in _sys.modules, (+ "test premise broken — module isn't in sys.modules"+ )+ assert "appsync_events" not in _loaded_modules, (+ "test premise broken — module is still in _loaded_modules"+ )++ save_dict = _build_persistence_save_dict()++ assert "appsync_events" in save_dict, (+ "shutdown save loop dropped appsync_events even though the "+ "module was imported via a sibling import. The sys.modules "+ "fallback in `_build_persistence_save_dict` is missing or broken."+ )+ # The value must be the bound get_state method, not the result.+ assert callable(save_dict["appsync_events"]), (+ "save_dict should map to a callable (get_state method ref) — "+ "save_all invokes it. Got %r" % (save_dict["appsync_events"],)+ )+ finally:+ if saved is not None:+ _loaded_modules["appsync_events"] = saved+++def test_save_dict_skips_modules_never_imported():+ """The sys.modules fallback must NOT save state for modules that+ were never imported at all — there's no state to capture and any+ `get_state()` call on a non-imported module would attribute-error.+ Defensive guard: ensure the fallback path's `hasattr` check works."""+ import sys as _sys++ from ministack.app import _build_persistence_save_dict, _loaded_modules, _state_map++ # Pick any persisted module and ensure it's truly absent from both+ # `_loaded_modules` and `sys.modules`. `cur` is an obscure one that+ # most test sessions won't have touched.+ target = "ecs_metadata" # not in _state_map → guaranteed absent from save_dict+ assert target not in {v for v in _state_map.values()}, (+ "test premise broken — pick a module not in _state_map"+ )++ # Even after the fallback path runs, ecs_metadata must not appear.+ save_dict = _build_persistence_save_dict()+ assert "ecs_metadata" not in save_dict, (+ "save_dict picked up a module that isn't even in _state_map — "+ "the loop's key-membership check is broken."+ )++# ── Functional round-trip tests ────────────────────────────────────────def _round_trip(mod_name, svc_key, populate_fn, observe_fn):diff --git a/tests/test_rds.py b/tests/test_rds.pyindex bcf7d350..f70503a6 100644--- a/tests/test_rds.py+++ b/tests/test_rds.py@@ -1562,14 +1562,18 @@ def test_rds_restore_state_removes_stale_container_before_respawn(monkeypatch):def reload(self): passdef stop(self, timeout=2): pass- def remove(self, v=False):+ def remove(self, **kwargs):removed.append(self.name)stale = FakeContainer(name="ministack-rds-stale-db", container_id="cid-stale")class FakeContainers:def get(self, name):- if name == "ministack-rds-stale-db":+ # First call returns the stale container; after .remove() is+ # called and `removed` is populated, subsequent .get()s for+ # the same name raise "not found" — mirrors real docker after+ # a successful force-remove.+ if name == "ministack-rds-stale-db" and "ministack-rds-stale-db" not in removed:return staleraise Exception("not found")@@ -1615,6 +1619,256 @@ def test_rds_restore_state_removes_stale_container_before_respawn(monkeypatch):m._instances.clear()+def test_rds_respawn_does_not_bind_engine_port_on_host(monkeypatch):+ """Real AWS reports `Endpoint.Port` as the engine's standard port+ (5432 for postgres) regardless of the docker host port mapping.+ Respawn must NOT read `Endpoint.Port` as the host bind port — doing+ so makes every restart try to bind 0.0.0.0:5432 and collide.+ Regression for the bug doodaz reported on #692 after 1.3.48: the+ 1.3.47 + 1.3.48 fixes covered restore-then-respawn but left this+ port-reuse bug live."""+ from ministack.services import rds as m++ runs = []++ class FakeContainer:+ def __init__(self, name):+ self.id = "cid-fake"+ self.name = name+ self.attrs = {"NetworkSettings": {"Networks": {}}}+ def reload(self): pass+ def stop(self, timeout=2): pass+ def remove(self, v=False): pass++ class FakeContainers:+ def get(self, name): raise Exception("not found")+ def run(self, **kwargs):+ runs.append(kwargs)+ return FakeContainer(kwargs["name"])++ class FakeDocker:+ def __init__(self):+ self.containers = FakeContainers()++ monkeypatch.setattr(m, "_get_docker", lambda: FakeDocker())+ monkeypatch.setattr(m, "_get_ministack_network", lambda c: None)++ db_id = "port-collision-db"+ # Simulate state as persisted by a real run: Endpoint.Port has been+ # overwritten to the container port (5432 for postgres) to match AWS.+ persisted_state = {+ "instances": {db_id: {+ "DBInstanceIdentifier": db_id,+ "Engine": "postgres",+ "EngineVersion": "16.3",+ "MasterUsername": "admin",+ "_MasterUserPassword": "password123",+ "DBName": "mydb",+ "DBInstanceStatus": "available",+ "Endpoint": {"Address": "localhost", "Port": 5432, "HostedZoneId": "Z"},+ }},+ "clusters": {}, "subnet_groups": {}, "param_groups": {},+ "snapshots": {}, "db_cluster_param_groups": {},+ "db_cluster_snapshots": {}, "option_groups": {},+ "global_clusters": {}, "tags": {}, "port_counter": 15500,+ }++ m._instances.clear()+ m.restore_state(persisted_state)++ deadline = time.time() + 5+ while time.time() < deadline and not runs:+ time.sleep(0.05)++ assert runs, "restore_state did not respawn the Docker container"+ port_mapping = runs[0]["ports"]+ # ports == {"5432/tcp": host_port} — host_port must NOT be 5432.+ assert port_mapping == {"5432/tcp": runs[0]["ports"]["5432/tcp"]}, port_mapping+ host_port = port_mapping["5432/tcp"]+ assert host_port != 5432, (+ f"respawn tried to bind container port 5432 to host port 5432 — "+ f"will collide with anything else listening on 5432. "+ f"ports={port_mapping}"+ )+ assert host_port >= 15432, (+ f"respawn host port {host_port} not in MiniStack's allocated range "+ f"(>=15432) — looks like Endpoint.Port leaked through again."+ )++ # The instance must now carry _HostPort so subsequent respawns reuse+ # the same host port instead of allocating a new one each time.+ restored = m._instances.get(db_id)+ assert restored.get("_HostPort") == host_port, (+ "respawn did not persist _HostPort on the instance — next restart "+ "will pick a different port and break clients with cached connection strings."+ )++ m._instances.clear()+++def test_rds_next_port_skips_busy_ports(monkeypatch):+ """`_next_port` must probe each candidate and skip ports already+ bound on the host. Without this, a counter-only allocator hands out+ a port that `docker run` will immediately fail to bind."""+ from ministack.services import rds as m++ busy_ports = {15432, 15433, 15434}+ monkeypatch.setattr(m, "_is_host_port_free", lambda p: p not in busy_ports)+ monkeypatch.setattr(m, "_port_counter", [15432])++ port = m._next_port()+ assert port == 15435, (+ f"_next_port returned {port} but ports 15432-15434 were busy — "+ f"it must skip taken ports, not blindly hand them out."+ )+++def test_rds_respawn_falls_back_when_persisted_host_port_taken(monkeypatch):+ """If `_HostPort` from persisted state is taken by another process+ on the host, respawn must fall back to a fresh free port instead+ of trying to bind a port we know is unavailable."""+ from ministack.services import rds as m++ runs = []++ class FakeContainer:+ def __init__(self, name):+ self.id = "cid-fake"; self.name = name+ self.attrs = {"NetworkSettings": {"Networks": {}}}+ def reload(self): pass+ def stop(self, timeout=2): pass+ def remove(self, **kwargs): pass++ class FakeContainers:+ def get(self, name): raise Exception("not found")+ def run(self, **kwargs):+ runs.append(kwargs)+ return FakeContainer(kwargs["name"])++ class FakeDocker:+ def __init__(self): self.containers = FakeContainers()++ monkeypatch.setattr(m, "_get_docker", lambda: FakeDocker())+ monkeypatch.setattr(m, "_get_ministack_network", lambda c: None)+ # Persisted port 15500 is "taken"; anything else is free.+ monkeypatch.setattr(m, "_is_host_port_free", lambda p: p != 15500)+ monkeypatch.setattr(m, "_port_counter", [15600])++ db_id = "fallback-db"+ persisted_state = {+ "instances": {db_id: {+ "DBInstanceIdentifier": db_id,+ "Engine": "postgres",+ "EngineVersion": "16.3",+ "MasterUsername": "admin",+ "_MasterUserPassword": "p",+ "DBName": "mydb",+ "DBInstanceStatus": "available",+ "Endpoint": {"Address": "localhost", "Port": 5432, "HostedZoneId": "Z"},+ "_HostPort": 15500,+ }},+ "clusters": {}, "subnet_groups": {}, "param_groups": {},+ "snapshots": {}, "db_cluster_param_groups": {},+ "db_cluster_snapshots": {}, "option_groups": {},+ "global_clusters": {}, "tags": {}, "port_counter": 15600,+ }++ m._instances.clear()+ m.restore_state(persisted_state)++ deadline = time.time() + 5+ while time.time() < deadline and not runs:+ time.sleep(0.05)++ assert runs, "respawn never happened"+ host_port = runs[0]["ports"]["5432/tcp"]+ assert host_port != 15500, (+ f"respawn used stored _HostPort=15500 despite it being busy — "+ f"will fail at docker bind. Got host_port={host_port}."+ )+ assert m._instances[db_id]["_HostPort"] == host_port, (+ "fallback port not persisted on instance — next restart will "+ "try the same busy port again."+ )++ m._instances.clear()+++def test_rds_respawn_force_removes_stale_created_container(monkeypatch):+ """Doodaz observed a half-spawned `Created`-status container with+ the deterministic name blocking respawn. Plain `.remove()` doesn't+ handle running/created containers; respawn must use `force=True`."""+ from ministack.services import rds as m++ remove_calls = []++ class FakeStaleContainer:+ def remove(self, **kwargs):+ remove_calls.append(kwargs)+ if not kwargs.get("force"):+ raise Exception("cannot remove non-stopped container without force")++ class FakeContainer:+ def __init__(self, name):+ self.id = "cid-fake"; self.name = name+ self.attrs = {"NetworkSettings": {"Networks": {}}}+ def reload(self): pass+ def stop(self, timeout=2): pass+ def remove(self, **kwargs): pass++ name_present = {"yes": True}++ class FakeContainers:+ def get(self, name):+ if name_present["yes"]:+ name_present["yes"] = False # second .get() (verification) returns "not found"+ return FakeStaleContainer()+ raise Exception("not found")+ def run(self, **kwargs):+ return FakeContainer(kwargs["name"])++ class FakeDocker:+ def __init__(self): self.containers = FakeContainers()++ monkeypatch.setattr(m, "_get_docker", lambda: FakeDocker())+ monkeypatch.setattr(m, "_get_ministack_network", lambda c: None)+ monkeypatch.setattr(m, "_is_host_port_free", lambda p: True)+ monkeypatch.setattr(m, "_port_counter", [15700])++ db_id = "stale-created-db"+ persisted_state = {+ "instances": {db_id: {+ "DBInstanceIdentifier": db_id,+ "Engine": "postgres",+ "EngineVersion": "16.3",+ "MasterUsername": "admin",+ "_MasterUserPassword": "p",+ "DBName": "mydb",+ "DBInstanceStatus": "available",+ "Endpoint": {"Address": "localhost", "Port": 5432, "HostedZoneId": "Z"},+ }},+ "clusters": {}, "subnet_groups": {}, "param_groups": {},+ "snapshots": {}, "db_cluster_param_groups": {},+ "db_cluster_snapshots": {}, "option_groups": {},+ "global_clusters": {}, "tags": {}, "port_counter": 15700,+ }++ m._instances.clear()+ m.restore_state(persisted_state)++ deadline = time.time() + 5+ while time.time() < deadline and m._instances.get(db_id, {}).get("DBInstanceStatus") not in ("available", "failed"):+ time.sleep(0.05)++ assert remove_calls, "respawn did not attempt to remove stale container"+ assert any(c.get("force") for c in remove_calls), (+ f"respawn called .remove() without force=True (calls={remove_calls}) — "+ f"will fail on Created/Running containers like the one doodaz hit."+ )++ m._instances.clear()++# ========== from test_rds_lambda_network.py ==========# RDS+Lambda network reachability via DOCKER_NETWORK auto-detect.import io