Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
feat(concurrency): route blocking work by whether it can re-enter the server
Nahuel Nucera9bfa0604bbae
diff --git a/ministack/app.py b/ministack/app.pyindex edefc34c..29ec527d 100644--- a/ministack/app.py+++ b/ministack/app.py@@ -161,6 +161,8 @@ _NON_S3_VHOST_NAMES = frozenset({"inspector2", "dsql",})+from ministack.core import container_reaper+from ministack.core.concurrency import spawn_backgroundfrom ministack.core.hypercorn_compat import install as _install_hypercorn_compatfrom ministack.core.persistence import PERSIST_STATE, load_state, save_allfrom ministack.core.responses import _12_DIGIT_RE, set_request_account_id, set_request_region@@ -231,6 +233,48 @@ def _get_module(name: str):return mod+# The loop serving HTTP requests, captured at lifespan startup. Worker threads+# (Step Functions executions, CloudFormation provisioners) need it to call a+# service handler, because a handler may genuinely await now.+_MAIN_LOOP = None+++def call_service_handler_sync(handler, method, path, headers, body, query_params,+ timeout: float | None = None):+ """Call an async service handler from a worker thread and return its result.++ In-process callers used to drive handlers with ``coro.send(None)``, relying+ on an unwritten invariant that service handlers never actually await. That+ held only while every handler was synchronous underneath. The moment one+ awaits — offloading Docker work, for instance — the first step runs with no+ running event loop and raises ``RuntimeError: no running event loop``, which+ escapes past the caller's ``except StopIteration``.++ So the coroutine is scheduled on the loop that is actually running and this+ thread waits for the result. Callers must be on a worker thread: waiting on+ the loop from the loop itself would deadlock, so that is refused loudly+ rather than hanging.+ """+ loop = _MAIN_LOOP+ if loop is None or loop.is_closed():+ # No server running (unit tests, CLI use): drive it on a private loop.+ return asyncio.run(handler(method, path, headers, body, query_params))++ try:+ running = asyncio.get_running_loop()+ except RuntimeError:+ running = None+ if running is loop:+ raise RuntimeError(+ "call_service_handler_sync must be called from a worker thread, "+ "not from the event loop — await the handler directly instead"+ )++ future = asyncio.run_coroutine_threadsafe(+ handler(method, path, headers, body, query_params), loop)+ return future.result(timeout)++def _lazy_handler(module_name: str):"""Return a callable that lazily imports module_name and delegates to handle_request."""@@ -1988,6 +2032,8 @@ async def _handle_lifespan(scope, receive, send):import concurrent.futures_max_workers = int(os.environ.get("MINISTACK_WORKER_THREADS", "64"))+ global _MAIN_LOOP+ _MAIN_LOOP = asyncio.get_running_loop()asyncio.get_running_loop().set_default_executor(concurrent.futures.ThreadPoolExecutor(max_workers=_max_workers,@@ -2000,7 +2046,28 @@ async def _handle_lifespan(scope, receive, send):# process. Persistence strips container ids from snapshots, so any# ministack-labelled container alive at boot is by definition an# orphan whose name will collide on next create.- _stop_docker_containers()+ #+ # Bounded: the sweep normally finishes in well under a second and we+ # wait for it, so a create right after boot still sees a clean slate.+ # But it talks to the Docker daemon, and a slow or wedged daemon must+ # never stop MiniStack from binding its port — that presents as a+ # silent hang with an empty log. Past the deadline it finishes in the+ # background while the server comes up.+ # Periodic reclamation for the whole process lifetime, not just at+ # boot: without it, containers (and their anonymous volumes) pile up+ # for as long as MiniStack runs.+ container_reaper.start(_reaper_docker_client)+ # Boot only: also reclaim ownerless leftovers from a MiniStack that+ # predates the instance labels. Holding the gateway port means any+ # such container is ours. Shutdown deliberately does not do this —+ # another instance may be live on this daemon.+ _reap = spawn_background(_stop_docker_containers, True,+ thread_name="ministack-boot-reap")+ _reap.join(timeout=_DOCKER_REAP_BOOT_DEADLINE)+ if _reap.is_alive():+ logger.warning(+ "Docker orphan reap still running after %ss; continuing "+ "startup without it", _DOCKER_REAP_BOOT_DEADLINE)if PERSIST_STATE:_load_persisted_state()# Start the Transfer Family SFTP listener after persistence is@@ -2010,14 +2077,21 @@ async def _handle_lifespan(scope, receive, send):# — its top-level `import asyncssh` pulls cryptography+OpenSSL# (~2–4 MiB of heap, plus C-level SSL contexts) which is pure# overhead for callers that aren't using Transfer Family.+ # Only bind at boot for servers restored from persistence, which a+ # client can connect to without calling CreateServer first. A fresh+ # boot binds lazily on CreateServer instead — nobody can connect to+ # a server that does not exist yet — which keeps asyncssh (and the+ # cryptography + OpenSSL it drags in, ~26 MB of heap, measured+ # 39.7 -> 27.9 MiB idle RSS) out of every MiniStack that never+ # touches Transfer Family. Guarded on the module already being+ # loaded, mirroring dsql below, so we never import it just to ask._sftp_env = os.environ.get("SFTP_ENABLED", "").strip().lower()+ _transfer_mod = _loaded_modules.get("transfer")if _sftp_env in ("0", "false", "no", "off"):logger.debug("SFTP_ENABLED=%s — skipping transfer module import.", _sftp_env)- else:+ elif _transfer_mod is not None and _transfer_mod.has_servers():try:- from ministack.services import transfer-- await transfer.sftp_start()+ await _transfer_mod.sftp_start()except Exception as e:logger.warning("Transfer SFTP startup failed: %s", e)# Start the IoT mTLS MQTT listener, for the same reason and in the@@ -2093,36 +2167,43 @@ async def _handle_lifespan(scope, receive, send):return-def _stop_docker_containers():- """Stop all Docker containers managed by MiniStack (RDS, ECS, ElastiCache).- Uses container labels to find them — does not touch service state.+# Docker orphan reap bounds. The client timeout caps any single daemon call;+# the boot deadline caps how long startup will wait for the whole sweep before+# letting the server bind and finishing the reap in the background.+_DOCKER_REAP_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))+_DOCKER_REAP_BOOT_DEADLINE = 10.0- Skip entirely if no Docker socket is available: importing the docker- SDK (and its requests/urllib3/idna transitive deps) costs ~1 MiB of- Python heap before we even know whether there's anything to clean.- """++def _reaper_docker_client():+ """Docker client for the periodic reaper, or None when there is no daemon."""sock = os.environ.get("DOCKER_HOST") or "unix:///var/run/docker.sock"- if sock.startswith("unix://"):- sock_path = sock[len("unix://"):]- if not os.path.exists(sock_path):- return+ if sock.startswith("unix://") and not os.path.exists(sock[len("unix://"):]):+ return Nonetry:import docker- client = docker.from_env()+ return docker.from_env(timeout=_DOCKER_REAP_TIMEOUT)except Exception:+ return None+++def _stop_docker_containers(include_unlabelled: bool = False):+ """Remove every MiniStack container. Boot and shutdown only.++ The reclamation rules live in ``core.container_reaper``: this is the+ process-boundary phase (everything goes), while the periodic pass while the+ server is live is ownership-aware and much more conservative. Keeping both+ in one module means one label list and one place to reason about which+ containers are ours.++ Skips entirely when there is no Docker socket: importing the docker SDK+ (and its requests/urllib3/idna transitive deps) costs ~1 MiB of Python heap+ before we even know whether there is anything to clean.+ """+ client = _reaper_docker_client()+ if client is None:return- for label in ("ministack=rds", "ministack=ecs", "ministack=elasticache", "ministack=eks", "ministack=lambda", "ministack=dsql"):- try:- # all=True so exited-but-not-removed orphans get cleaned at boot.- for c in client.containers.list(all=True, filters={"label": label}):- try:- c.stop(timeout=5)- c.remove(v=True)- except Exception:- pass- except Exception:- pass+ container_reaper.reap_all(client, include_unlabelled=include_unlabelled)def _build_persistence_save_dict():diff --git a/ministack/core/concurrency.py b/ministack/core/concurrency.pynew file mode 100644index 00000000..a04b79e5--- /dev/null+++ b/ministack/core/concurrency.py@@ -0,0 +1,220 @@+"""Off-loop execution primitives.++MiniStack is a single ASGI process, so every blocking call has to be placed+deliberately. The rule is not "which service is this" but **can this work+re-enter the server**:++- Work that cannot re-enter (an outbound HTTP call to a real endpoint, a+ DuckDB query, a zip extraction) is safe on a bounded shared pool. Use+ :func:`run_offloop`.+- Work that CAN re-enter (anything that runs a Lambda handler, a Docker+ container that talks back to 4566, a CloudFormation custom resource+ blocking on its ResponseURL callback) must never take a slot from a bounded+ pool: the nested request needs a slot of its own, and it queues behind the+ very call waiting on it. Deadlock arrives at exactly the pool size. Use+ :func:`run_reentrant`, which gives each such call its own thread.+- Fire-and-forget work (delivery loops, reapers, background provisioning) uses+ :func:`spawn_background` so it is named, its exceptions are logged, and it is+ visible to :func:`active_counts` instead of being an anonymous+ ``threading.Thread``.++Nothing blocking belongs on the event loop itself. A blocking call there+freezes every service at once, which is strictly worse than exhausting a pool.++Bounding in-flight re-entrant work is a *policy* question that belongs to the+caller (Lambda enforces AWS's concurrency model and throttles), not to this+module: a blocking gate here would re-create the deadlock it exists to remove.++Moving handlers off the loop also removed the serialisation they were silently+relying on. Under the loop, read-modify-write on a service record could not+interleave; off it, every ``if record still exists: publish into it`` became a+race, and a delete landing in the gap leaves a container nothing references. Use+:func:`resource_lock` for those, under one rule: **hold it across a record+mutation, never across a Docker call or a re-entrant dispatch.** A lock held+across either is how #1277's per-service locking deadlocked.+"""++import asyncio+import contextvars+import logging+import threading++logger = logging.getLogger("concurrency")++# Live counters, for diagnostics via /_ministack/health.+_counts_lock = threading.Lock()+_reentrant_threads = 0+_background_threads = 0+++def active_counts() -> dict:+ """Snapshot of threads this module currently has running."""+ with _counts_lock:+ return {"reentrant": _reentrant_threads, "background": _background_threads}+++_resource_locks: dict = {}+_resource_locks_guard = threading.Lock()+++def resource_lock(scope: str, key: str) -> threading.Lock:+ """A lock for one resource — e.g. ``resource_lock("elasticache", cluster_id)``.++ For making a read-modify-write on a service record atomic now that handlers+ no longer run serialised on the event loop. Typical use is a deferred+ provisioner publishing what it started, versus a delete removing the record:++ with resource_lock("elasticache", cluster_id): # short, no I/O+ rec = _clusters.get(cluster_id)+ stale = rec is not record or rec.get("_deleting")+ if not stale:+ rec["_docker_container_id"] = container_id+ if stale:+ tear_down(container_id) # outside the lock++ Two rules keep this safe:++ - **Never hold it across a Docker call or anything that can re-enter+ MiniStack.** That is what makes per-service locking deadlock: a container+ MiniStack starts is handed ``AWS_ENDPOINT_URL`` pointing back at MiniStack,+ so it calls in while the starter still holds the lock.+ - **Compare record identity, not truthiness.** ``if rec:`` cannot tell a+ surviving record from a same-named one created after a delete; ``rec is+ record`` can.++ Locks are kept for the process lifetime. They are one mutex per resource id+ ever seen, which is far cheaper than the bookkeeping to reclaim them safely.+ """+ with _resource_locks_guard:+ return _resource_locks.setdefault((scope, key), threading.Lock())+++def _bump(kind: str, delta: int) -> None:+ global _reentrant_threads, _background_threads+ with _counts_lock:+ if kind == "reentrant":+ _reentrant_threads += delta+ else:+ _background_threads += delta+++async def run_offloop(fn, *args, **kwargs):+ """Run blocking work that cannot re-enter the server, on the shared pool.++ Thin alias for :func:`asyncio.to_thread` so call sites state their+ classification explicitly and an audit can distinguish "reviewed, cannot+ re-enter" from "nobody thought about it".+ """+ return await asyncio.to_thread(fn, *args, **kwargs)+++async def run_reentrant(fn, *args, thread_name: str = "ministack-reentrant"):+ """Run blocking work that may call back into this server, on its own thread.++ A dedicated thread per call, not a pooled worker: nested requests draw from+ the same pool, so a bound of any size only moves the deadlock edge rather+ than removing it.++ ContextVars propagate as ``asyncio.to_thread`` would.++ Cancellation defers to completion. A thread cannot be cancelled once it is+ running, so if a disconnected client let the awaiter unwind immediately, the+ caller would proceed — releasing a lease, taking a reset barrier, tearing+ down request state — while the worker is still mutating that same state. So+ the wait is shielded, repeated cancellation is absorbed until the worker+ finishes, and the original ``CancelledError`` is re-raised afterwards. The+ awaiting task is suspended, not the loop, so this costs nothing but the+ task's own latency.+ """+ loop = asyncio.get_running_loop()+ future: asyncio.Future = loop.create_future()+ ctx = contextvars.copy_context()++ def _deliver(result, exc):+ if not future.done():+ if exc is not None:+ future.set_exception(exc)+ else:+ future.set_result(result)++ def _post(result, exc):+ try:+ loop.call_soon_threadsafe(_deliver, result, exc)+ except RuntimeError:+ pass # loop already closed (shutdown) — nobody left to deliver to++ def _run():+ result, exc = None, None+ try:+ result = ctx.run(fn, *args)+ except Exception as e: # noqa: BLE001 — delivered to the awaiter+ exc = e+ except BaseException as e:+ # SystemExit / KeyboardInterrupt: hand it to the awaiter so the+ # request fails instead of hanging on a future nothing resolves,+ # then let it unwind this thread rather than swallowing it.+ _post(None, e)+ _bump("reentrant", -1)+ raise+ _post(result, exc)+ _bump("reentrant", -1)++ _bump("reentrant", 1)+ try:+ threading.Thread(target=_run, daemon=True, name=thread_name).start()+ except RuntimeError as exc:+ _bump("reentrant", -1)+ # OS thread limit. Fail the request loudly instead of awaiting a future+ # nothing will ever resolve.+ raise RuntimeError(f"cannot start {thread_name} thread: {exc}") from exc++ try:+ return await asyncio.shield(future)+ except asyncio.CancelledError as cancellation:+ while not future.done():+ try:+ await asyncio.shield(future)+ except asyncio.CancelledError:+ continue # absorb repeated cancellation; the worker is still live+ except Exception:+ break # the worker failed; that is a completion too+ # Consume a failed worker's exception so asyncio does not report it as+ # never-retrieved after the awaiting request has already gone away.+ if future.done() and not future.cancelled():+ try:+ future.result()+ except Exception:+ pass+ raise cancellation+++def spawn_background(fn, *args, thread_name: str = "ministack-background", **kwargs):+ """Start named fire-and-forget work with its exceptions logged.++ Replaces bare ``threading.Thread(target=..., daemon=True).start()`` so that+ background failures surface in the log instead of dying silently, and so+ the thread is countable.+ """+ ctx = contextvars.copy_context()++ def _run():+ try:+ ctx.run(lambda: fn(*args, **kwargs))+ except Exception:+ logger.exception("background task %s failed", thread_name)+ finally:+ _bump("background", -1)++ _bump("background", 1)+ thread = threading.Thread(target=_run, daemon=True, name=thread_name)+ try:+ thread.start()+ except RuntimeError:+ # OS thread limit. `_run` never executes, so its `finally` never runs+ # and the counter would leak — permanently, and precisely when someone+ # is reading it to diagnose thread exhaustion. Callers that can fall+ # back to running inline (firehose's Iceberg delivery) catch this, so+ # it must still propagate. `run_reentrant` does the same.+ _bump("background", -1)+ raise+ return threaddiff --git a/ministack/core/container_reaper.py b/ministack/core/container_reaper.pynew file mode 100644index 00000000..2a91682b--- /dev/null+++ b/ministack/core/container_reaper.py@@ -0,0 +1,281 @@+"""Periodic reclamation of MiniStack's Docker containers.++Without this, a long-running MiniStack accumulates containers — and the+anonymous volumes they carry — for its whole lifetime. Only ECS had a reaper;+RDS, EKS, ElastiCache, MWAA and OpenSearch containers survived until the process+restarted, and the boot sweep then paid for all of them at once (measured: 119s+on a healthy daemon with a session's worth of orphans).++The rule is deliberately conservative, because a *stopped* container is not+necessarily garbage: ``StopDBInstance`` leaves an exited container that+``StartDBInstance`` must be able to restart. So a container is only reclaimed+when it is unambiguously garbage:++- state is ``created`` or ``dead`` — never actually ran, or the daemon lost it;+- or it is ``exited`` **and** no live service record still references its id.++Services publish the ids they still own via ``register_live_ids``. A service+that does not register is never reaped from here — silence means "unknown", not+"garbage".++Reaping is also scoped to *this* MiniStack. Several instances can share a Docker+daemon, and "not in my records" says nothing about another instance's containers:+a ``StopDBCluster``-ed database is exited by design and expected to restart, so a+second gateway would delete it after the grace period. Every container carries+``ministack.instance`` — the gateway address, which only one process can hold, so+it is an identity rather than a guess — and ``ministack.boot``, a per-process+nonce. The periodic pass matches both, so it cannot see another instance's+containers at all; the boot sweep matches the address only, since holding the+port proves the previous owner is gone.+"""++import concurrent.futures+import logging+import os+import socket+import threading+import time+import uuid++logger = logging.getLogger("container_reaper")++REAP_INTERVAL = 60.0+# An exited container gets this long before it is considered abandoned, so a+# service that is mid-restart is not raced.+EXITED_GRACE = 120.0++# Label carrying which MiniStack owns a container, and which run of it.+INSTANCE_LABEL = "ministack.instance"+BOOT_LABEL = "ministack.boot"++_providers: dict = {}+_lock = threading.Lock()+_started = False+_boot_nonce = uuid.uuid4().hex[:12]+++def instance_id() -> str:+ """Identity of this MiniStack: the address it serves on.++ The gateway port is the one thing that is provably exclusive — two instances+ cannot bind it — so it identifies an instance without any coordination.+ """+ host = os.environ.get("MINISTACK_HOSTNAME") or socket.gethostname()+ port = os.environ.get("GATEWAY_PORT") or os.environ.get("EDGE_PORT") or "4566"+ return f"{host}:{port}"+++def own_labels(service: str, **extra) -> dict:+ """Labels every MiniStack-created container must carry.++ ``service`` is the value of the ``ministack`` label (e.g. "rds"). Anything in+ ``extra`` is merged, so callers keep their own labels.+ """+ labels = {+ "ministack": service,+ INSTANCE_LABEL: instance_id(),+ BOOT_LABEL: _boot_nonce,+ }+ labels.update({k: v for k, v in extra.items() if v is not None})+ return labels+++def register_live_ids(label: str, provider) -> None:+ """Register a callable returning the container ids ``label`` still owns.++ ``label`` is the value of the container's ``ministack`` label (e.g. "rds").+ The provider must return an iterable of container ids; anything it omits+ with an exited container is treated as abandoned.+ """+ with _lock:+ _providers[label] = provider+++def _live_ids() -> tuple[set, set]:+ """(ids still owned, labels that answered) — labels that don't answer are skipped."""+ live, known = set(), set()+ with _lock:+ providers = dict(_providers)+ for label, provider in providers.items():+ try:+ ids = provider() or ()+ except Exception as exc:+ logger.debug("reaper: provider for %s failed: %s", label, exc)+ continue+ known.add(label)+ live.update(i for i in ids if i)+ return live, known+++def reap_abandoned(docker_client) -> int:+ """Remove abandoned containers. Returns how many were reclaimed."""+ if docker_client is None:+ return 0+ live, known = _live_ids()+ now = time.time()+ removed = 0+ try:+ # Scoped to this instance *and* this run of it. Another MiniStack's+ # containers are not merely skipped later — they are never listed, so no+ # amount of downstream logic can reclaim them. Containers predating the+ # labels are likewise invisible here; the boot sweep still gets them.+ containers = docker_client.containers.list(all=True, filters={"label": [+ f"{INSTANCE_LABEL}={instance_id()}",+ f"{BOOT_LABEL}={_boot_nonce}",+ ]})+ except Exception as exc:+ logger.debug("reaper: listing containers failed: %s", exc)+ return 0++ for c in containers:+ try:+ label = (c.labels or {}).get("ministack", "")+ status = c.status+ if status not in ("created", "dead", "exited"):+ continue # running / restarting / paused: not ours to judge+ if label not in known:+ # No provider for this service, so nothing can vouch for the+ # container. Silence means "unknown", not "garbage" — a service+ # that creates a container and starts it a moment later (ECS+ # tasks pass through `created`) must never be raced.+ continue+ if c.id in live:+ continue # a stopped-but-live resource (e.g. StopDBInstance)+ finished = (c.attrs.get("State", {}) or {}).get("FinishedAt", "")+ if finished and _age_seconds(finished, now) < EXITED_GRACE:+ continue+ if status == "created":+ # `created` has no FinishedAt; use Created to age it out so a+ # container mid-start is never taken from under its service.+ created_at = c.attrs.get("Created", "")+ if created_at and _age_seconds(created_at, now) < EXITED_GRACE:+ continue+ c.remove(force=True, v=True) # v=True: reclaim the anonymous volume too+ removed += 1+ except Exception:+ continue+ if removed:+ logger.info("Reaped %d abandoned container(s)", removed)+ return removed+++def _age_seconds(finished_at: str, now: float) -> float:+ """Seconds since a Docker RFC3339 timestamp; 0 if it cannot be parsed."""+ import datetime+ try:+ ts = finished_at.replace("Z", "+00:00")+ if "." in ts:+ head, rest = ts.split(".", 1)+ frac, _, tz = rest.partition("+")+ ts = f"{head}.{frac[:6]}+{tz}" if tz else f"{head}.{frac[:6]}"+ return max(0.0, now - datetime.datetime.fromisoformat(ts).timestamp())+ except Exception:+ return 0.0+++def start(get_docker) -> None:+ """Start the reaper thread once. ``get_docker`` is called each pass."""+ global _started+ with _lock:+ if _started:+ return+ _started = True++ def _loop():+ while True:+ time.sleep(REAP_INTERVAL)+ try:+ reap_abandoned(get_docker())+ except Exception as exc:+ logger.debug("reaper iteration failed: %s", exc)++ threading.Thread(target=_loop, daemon=True, name="ministack-container-reaper").start()++# Every label a service stamps on a container, plus the key-only backstops so a+# new service (or an older container predating its label) is still reclaimed.+# One list, one place: the boot/shutdown sweep and the periodic pass both use it.+SERVICE_LABELS = (+ "ministack=rds", "ministack=ecs", "ministack=elasticache", "ministack=eks",+ "ministack=lambda", "ministack=dsql", "ministack=mwaa", "ministack=glue",+ "ministack=codebuild", "ministack=opensearch",+ "ministack", "com.ministack.service",+)+++def reap_all(docker_client, stop_timeout: int = 2, include_unlabelled: bool = False) -> int:+ """Remove every MiniStack container, whatever its state.++ For process boundaries only — boot and shutdown. At boot any surviving+ container **of ours** is by definition an orphan of a dead process: we hold+ the gateway port now, so whichever process created them is gone (persistence+ strips container ids from snapshots, so nothing can still own one). At+ shutdown everything is going away regardless. Neither assumption holds while+ the process is live, which is why :func:`reap_abandoned` exists separately+ and is far more conservative.++ Scoped by ``ministack.instance`` but *not* by boot nonce — the whole point is+ to reclaim a previous run's containers, which carry a different nonce.++ A container owned by a *different* instance is never reclaimed here, at boot+ or at shutdown. Shutdown is not a licence to sweep broadly: another MiniStack+ may be running on the same daemon, and taking its containers down as we exit+ is the very failure this scoping exists to prevent.++ ``include_unlabelled`` additionally matches containers that carry a MiniStack+ service label but *no* instance label — those predate this scheme and cannot+ be attributed to anyone. Boot uses it, because holding the gateway port means+ any such leftover is ours to clean; shutdown does not.+ """+ if docker_client is None:+ return 0+ mine = instance_id()+ seen, targets = set(), []++ def _collect(selector, accept):+ try:+ found = docker_client.containers.list(all=True, filters={"label": selector})+ except Exception as exc:+ logger.debug("reap_all: listing %s failed: %s", selector, exc)+ return+ for c in found:+ if c.id in seen or not accept(c):+ continue+ seen.add(c.id)+ targets.append(c)++ _collect([f"{INSTANCE_LABEL}={mine}"], lambda c: True)+ if include_unlabelled:+ for service_label in SERVICE_LABELS:+ # Only ownerless leftovers. A container carrying someone else's+ # instance label is off limits however it was found.+ _collect([service_label], lambda c: not (c.labels or {}).get(INSTANCE_LABEL))+ return drop_containers(targets, stop_timeout=stop_timeout)+++def drop_containers(containers, stop_timeout: int = 2, force: bool = False) -> int:+ """Stop and remove containers concurrently. Returns how many went away.++ Concurrent because ``stop`` is a per-container round trip that blocks until+ the container actually dies — serially that is O(n) seconds of wall clock.+ It matters because this runs on the ``/_ministack/reset`` path while the+ reset lock is held, so every other request queues behind it. Measured: a+ run that left 30 running ECS tasks took 54.4s serially, past the 45s the+ caller was willing to wait. The daemon parallelises stops fine; the cap only+ keeps a huge sweep from opening hundreds of sockets at once.+ """+ targets = [c for c in containers if c is not None]+ if not targets:+ return 0++ def _drop(c):+ try:+ c.stop(timeout=stop_timeout)+ c.remove(v=True, force=force) # v=True: reclaim the anonymous volume too+ return True+ except Exception:+ return False++ with concurrent.futures.ThreadPoolExecutor(+ max_workers=min(16, len(targets)), thread_name_prefix="ministack-reap") as pool:+ return sum(1 for ok in pool.map(_drop, targets) if ok)+diff --git a/ministack/core/lambda_runtime.py b/ministack/core/lambda_runtime.pyindex 113bb7f0..7559068f 100644--- a/ministack/core/lambda_runtime.py+++ b/ministack/core/lambda_runtime.py@@ -76,6 +76,18 @@ def _account_region_from_function_config(config: dict) -> tuple[str, str]:_workers: dict = {}_lock = threading.Lock()+# No per-function ceiling on warm subprocesses: AWS lets an unreserved function+# scale into the account pool, and capping width here would invent a throttle+# AWS never sends and diverge from the docker executor, which spawns freely.+# Concurrency is bounded once, at lambda_svc._acquire_execution_slot, via+# ReservedConcurrentExecutions and the existing LAMBDA_ACCOUNT_CONCURRENCY.+# The reaper reclaims the surplus subprocesses a burst leaves behind.+_LOCAL_MAX_WORKERS = 0+# Seconds a surplus worker may sit idle before it is reaped (the first worker+# per function is never reaped, so warm starts are unaffected). Shares the knob+# that already governs warm-container eviction rather than adding a second one.+_LOCAL_WORKER_TTL = float(os.environ.get("LAMBDA_WARM_TTL_SECONDS", "300"))+# ---------------------------------------------------------------------------# Recursive-loop lineage (runs inside every ministack-owned Python child)# ---------------------------------------------------------------------------@@ -935,6 +947,10 @@ class Worker:self._proc = Noneself._tmpdir = Noneself._lock = threading.Lock()+ # Leased out to an in-flight invocation. Guarded by the module ``_lock``+ # (see acquire_worker/release_worker), not by ``self._lock``.+ self.in_use = False+ self.last_used = time.time()self._cold = Trueself._start_time = Noneself._stderr_queue: queue.Queue = queue.Queue()@@ -1243,9 +1259,9 @@ class Worker:if reader.is_alive():# Timeout — kill the worker processlogger.warning("Lambda %s timed out after %ds — killing worker", self.func_name, timeout)- if self._proc:- self._proc.kill()- self._proc = None+ proc, self._proc = self._proc, None+ _signal(proc)+ _collect(proc, time.monotonic() + _REAP_GRACE)return {"status": "error","error": f"Task timed out after {timeout}.00 seconds",@@ -1259,37 +1275,195 @@ class Worker:response = result_box[0]if response.get("status") == "error":- if self._proc and self._proc.poll() is None:- self._proc.terminate()- self._proc = None+ proc, self._proc = self._proc, None+ _signal(proc)+ _collect(proc, time.monotonic() + _REAP_GRACE)response["cold_start"] = cold# Bounded drain — replaces the fixed 50ms sleep that was paid# by every warm invocation. Typical completion is 1–10ms.response["log"] = self._drain_stderr_bounded()return response- def kill(self):- if self._proc and self._proc.poll() is None:- self._proc.terminate()- self._proc = None+ def _discard_tmpdir(self):if self._tmpdir and os.path.exists(self._tmpdir):shutil.rmtree(self._tmpdir, ignore_errors=True)+ def kill(self):+ kill_workers([self])-def get_or_create_worker(func_name: str, config: dict, code_zip: bytes,- qualifier: str = "$LATEST") -> Worker:++# Seconds a worker gets to exit on SIGTERM before it is SIGKILLed. Paid once per+# *batch*, never once per worker — see kill_workers.+_REAP_GRACE = 2.0+++def _signal(proc) -> None:+ """Ask a worker subprocess to exit. Does not wait."""+ if proc is None:+ return+ try:+ if proc.poll() is None:+ proc.terminate()+ except Exception:+ pass+++def _collect(proc, deadline: float) -> None:+ """Wait for a signalled subprocess and reap it, escalating if it ignores SIGTERM.++ Collecting is the point. Signalling alone leaves a zombie: ``subprocess``+ only reaps an abandoned Popen lazily, when the *next* Popen is constructed,+ so an **idle** server keeps every zombie it made. On a host that is invisible+ — the count is small and something else is always spawning — but in a+ container MiniStack is PID 1: the zombies are its own children, nothing else+ will ever collect them, and they hold PID-table slots against the container's+ pid limit. Measured: a 320-invocation burst left 244 zombies that survived+ indefinitely while the server sat idle.++ Warm-worker pooling is what made this reachable. One worker per function was+ killed rarely; a pool of them is reaped by the hundred.+ """+ if proc is None:+ return+ try:+ proc.wait(timeout=max(0.0, deadline - time.monotonic()))+ except Exception:+ try:+ proc.kill()+ proc.wait(timeout=0.5)+ except Exception:+ pass # unkillable: leave it rather than block the caller+++def kill_workers(workers) -> None:+ """Terminate and reap a batch of workers, paying the grace period once.++ Two-phase on purpose. Signalling every worker first and only then waiting+ makes a mass teardown cost ``_REAP_GRACE`` in total; doing it worker by+ worker would cost ``n * _REAP_GRACE``, which on a pool built by a burst is+ minutes — and ``reset()`` used to pay it holding the global worker lock.+ """+ workers = [w for w in workers if w is not None]+ if not workers:+ return+ procs = []+ for worker in workers:+ proc, worker._proc = worker._proc, None+ procs.append(proc)+ _signal(proc)+ deadline = time.monotonic() + _REAP_GRACE+ for proc in procs:+ _collect(proc, deadline)+ for worker in workers:+ try:+ worker._discard_tmpdir()+ except Exception:+ pass+++def _worker_key(func_name: str, config: dict, qualifier: str) -> str:# Include account ID and region in the key to isolate workers across# accounts and regions. Two regions deploying the same function name must# not share a worker.account, region = _account_region_from_function_config(config)- key = f"{account}:{region}:{func_name}:{qualifier}"+ return f"{account}:{region}:{func_name}:{qualifier}"+++def acquire_worker(func_name: str, config: dict, code_zip: bytes,+ qualifier: str = "$LATEST", max_concurrency: int | None = None):+ """Lease a free execution environment for this function.++ Each key holds a *pool* of workers, mirroring how AWS gives concurrent+ invocations separate execution environments (and how the Docker executor's+ ``_warm_pool`` already behaves). A single shared worker per function could+ not: ``Worker.invoke`` holds the worker's lock for the whole handler, so a+ handler that invokes its own function — which real AWS supports, up to the+ ~16-hop recursive-loop limit — waited on the lock its own caller held and+ deadlocked. The thread the invocation runs on is irrelevant to that; the+ lock is the bottleneck.++ ``max_concurrency`` is an optional *resource* ceiling on how many warm+ subprocesses may exist for one function, not a concurrency limit — AWS+ concurrency is enforced upstream in ``lambda_svc._acquire_execution_slot``,+ once, for every executor. It defaults to unbounded, matching the docker+ executor. At a ceiling the caller gets ``(None, "func_cap")`` rather than a+ blocking wait, because waiting for a worker held by your own caller is the+ deadlock this exists to remove.++ Returns ``(worker, "reused"|"spawn")`` or ``(None, "func_cap")``. Release+ with :func:`release_worker`.+ """+ key = _worker_key(func_name, config, qualifier)+ cap = max_concurrency or _LOCAL_MAX_WORKERSwith _lock:- worker = _workers.get(key)- if worker is not None:- return worker+ entries = _workers.setdefault(key, [])+ for worker in entries:+ if not worker.in_use:+ worker.in_use = True+ worker.last_used = time.time()+ return worker, "reused"+ if cap and len(entries) >= cap:+ return None, "func_cap"worker = Worker(func_name, config, code_zip)- _workers[key] = worker- return worker+ worker.in_use = True+ worker.last_used = time.time()+ entries.append(worker)+ return worker, "spawn"+++def reap_idle_workers(ttl: float = None) -> int:+ """Kill workers idle longer than ``ttl`` and drop them from their pool.++ Without this the pools only ever grow: a burst of concurrency leaves its+ extra subprocesses alive forever. The first worker of each key is kept so+ the common warm-start path is unaffected.+ """+ ttl = _LOCAL_WORKER_TTL if ttl is None else ttl+ now = time.time()+ killed = []+ with _lock:+ for key, entries in list(_workers.items()):+ keep = []+ for i, worker in enumerate(entries):+ idle = now - getattr(worker, "last_used", now)+ if i == 0 or worker.in_use or idle < ttl:+ keep.append(worker)+ else:+ killed.append(worker)+ if keep:+ _workers[key] = keep+ else:+ _workers.pop(key, None)+ kill_workers(killed)+ return len(killed)+++def release_worker(worker: Worker) -> None:+ """Return a leased worker to its pool."""+ if worker is None:+ return+ with _lock:+ worker.in_use = False+++def workers_in_use() -> int:+ """Count of leased workers across every function, for concurrency limits."""+ with _lock:+ return sum(1 for lst in _workers.values() for w in lst if w.in_use)+++def get_or_create_worker(func_name: str, config: dict, code_zip: bytes,+ qualifier: str = "$LATEST") -> Worker:+ """Back-compat lease that never reports a cap and is released immediately.++ Retained for callers that hold a worker only for the duration of a single+ ``invoke`` on the calling thread. The lease is dropped before returning, so+ such callers still serialize on ``Worker._lock`` exactly as before rather+ than silently retaining a pool slot they never release.+ """+ worker, _reason = acquire_worker(func_name, config, code_zip, qualifier=qualifier)+ release_worker(worker)+ return workerdef invalidate_worker(func_name: str, qualifier: str = None,@@ -1324,16 +1498,16 @@ def invalidate_worker(func_name: str, qualifier: str = None,return Truewith _lock:- to_remove = [k for k in _workers if _matches(k)]- for k in to_remove:- worker = _workers.pop(k, None)- if worker:- worker.kill()+ doomed = [w for k in [k for k in _workers if _matches(k)]+ for w in (_workers.pop(k, []) or [])]+ # Outside the lock: terminating a batch takes as long as the slowest worker+ # takes to exit, and holding the pool lock for that blocks every invocation.+ kill_workers(doomed)def reset():"""Terminate all warm workers, clean up temp dirs, and clear the pool."""with _lock:- for worker in list(_workers.values()):- worker.kill()+ doomed = [w for entries in _workers.values() for w in entries]_workers.clear()+ kill_workers(doomed)diff --git a/ministack/core/responses.py b/ministack/core/responses.pyindex a706f6da..e81449e4 100644--- a/ministack/core/responses.py+++ b/ministack/core/responses.py@@ -131,13 +131,15 @@ class AccountScopedDict:return self._scoped(key) in self._datadef __len__(self):- return sum(1 for k in self._data if self._is_mine(k))+ # Snapshot: a concurrent insert during a live walk raises+ # RuntimeError("dictionary changed size during iteration").+ return sum(1 for k in list(self._data) if self._is_mine(k))def __bool__(self):- return any(self._is_mine(k) for k in self._data)+ return any(self._is_mine(k) for k in list(self._data))def __iter__(self):- for k in self._data:+ for k in list(self._data):if self._is_mine(k):yield self._unscope(k)@@ -163,13 +165,13 @@ class AccountScopedDict:return self._data.setdefault(self._scoped(key), default)def keys(self):- return [self._unscope(k) for k in self._data if self._is_mine(k)]+ return [self._unscope(k) for k in list(self._data) if self._is_mine(k)]def values(self):- return [v for k, v in self._data.items() if self._is_mine(k)]+ return [v for k, v in list(self._data.items()) if self._is_mine(k)]def items(self):- return [(self._unscope(k), v) for k, v in self._data.items() if self._is_mine(k)]+ return [(self._unscope(k), v) for k, v in list(self._data.items()) if self._is_mine(k)]def update(self, other):if isinstance(other, AccountScopedDict):@@ -241,13 +243,15 @@ class AccountRegionScopedDict:return self._scoped(key) in self._datadef __len__(self):- return sum(1 for k in self._data if self._is_mine(k))+ # Snapshot: a concurrent insert during a live walk raises+ # RuntimeError("dictionary changed size during iteration").+ return sum(1 for k in list(self._data) if self._is_mine(k))def __bool__(self):- return any(self._is_mine(k) for k in self._data)+ return any(self._is_mine(k) for k in list(self._data))def __iter__(self):- for k in self._data:+ for k in list(self._data):if self._is_mine(k):yield self._unscope(k)@@ -273,19 +277,19 @@ class AccountRegionScopedDict:return self._data.setdefault(self._scoped(key), default)def keys(self):- return [self._unscope(k) for k in self._data if self._is_mine(k)]+ return [self._unscope(k) for k in list(self._data) if self._is_mine(k)]def values(self):- return [v for k, v in self._data.items() if self._is_mine(k)]+ return [v for k, v in list(self._data.items()) if self._is_mine(k)]def items(self):- return [(self._unscope(k), v) for k, v in self._data.items() if self._is_mine(k)]+ return [(self._unscope(k), v) for k, v in list(self._data.items()) if self._is_mine(k)]def values_scoped(self, account_id, region):- return [v for k, v in self._data.items() if k[:2] == (account_id, region)]+ return [v for k, v in list(self._data.items()) if k[:2] == (account_id, region)]def items_scoped(self, account_id, region):- return [(self._unscope(k), v) for k, v in self._data.items() if k[:2] == (account_id, region)]+ return [(self._unscope(k), v) for k, v in list(self._data.items()) if k[:2] == (account_id, region)]def all_values(self):return list(self._data.values())diff --git a/ministack/services/alb.py b/ministack/services/alb.pyindex 8a796c93..8032a31a 100644--- a/ministack/services/alb.py+++ b/ministack/services/alb.py@@ -31,7 +31,8 @@ import timefrom urllib.parse import parse_qsfrom ministack.core.arn import ArnParseError, parse_arn-from ministack.core.persistence import PERSIST_STATE, load_state+from ministack.core.concurrency import run_reentrant+from ministack.core.persistence import load_statefrom ministack.core.responses import AccountRegionScopedDict, AccountScopedDict, get_account_id, get_region, new_uuidlogger = logging.getLogger("alb")@@ -1268,7 +1269,8 @@ async def _invoke_lambda_target(function_ref, tg_arn, method, path, headers, bod)return invocation_result- exec_result = await asyncio.to_thread(_invoke_and_record_metrics)+ exec_result = await run_reentrant(_invoke_and_record_metrics,+ thread_name="ministack-alb-invoke")lambda_response, _err = lambda_svc.lambda_execute_result_to_api_proxy_response(exec_result)if exec_result.get("error"):diff --git a/ministack/services/apigateway.py b/ministack/services/apigateway.pyindex 28e013ac..948efe7f 100644--- a/ministack/services/apigateway.py+++ b/ministack/services/apigateway.py@@ -52,10 +52,10 @@ import urllib.parseimport urllib.requestfrom ministack.core.arn import ArnParseError, parse_arn+from ministack.core.concurrency import run_reentrantfrom ministack.core.responses import (AccountRegionScopedDict,AccountScopedDict,- error_response_json,get_account_id,get_region,new_uuid,@@ -895,8 +895,9 @@ async def _invoke_request_authorizer_lambda(authorizer, event, account_id, regioif func_data is None or func_config is None:return Noneexec_record = lambda_svc._execution_record_for_config(func_data, func_config)- return await asyncio.to_thread(- lambda_svc._execute_function_with_config_scope, exec_record, event+ return await run_reentrant(+ lambda_svc._execute_function_with_config_scope, exec_record, event,+ thread_name="ministack-apigw-authorizer",)@@ -1593,7 +1594,8 @@ async def _invoke_lambda_proxy(# the shared helper so v1/v2 stay consistent and we get 429s on# ConcurrentInvocationLimitExceeded.exec_record = lambda_svc._execution_record_for_config(func_data, func_config)- result = await asyncio.to_thread(lambda_svc._execute_function_with_config_scope, exec_record, event)+ result = await run_reentrant(lambda_svc._execute_function_with_config_scope, exec_record, event,+ thread_name="ministack-apigw-invoke")lambda_response, _ = lambda_svc.lambda_execute_result_to_api_proxy_response(result)status = lambda_response.get("statusCode", 200)@@ -2582,7 +2584,7 @@ async def _invoke_ws_lambda(api_id: str, account_id: str, region: str, route: di)return worker.invoke(event, message_id)- result = await asyncio.to_thread(+ result = await run_reentrant(lambda_svc._run_with_function_config_scope,func_config,_invoke_worker,diff --git a/ministack/services/apigateway_v1.py b/ministack/services/apigateway_v1.pyindex eef75f6f..86f94d18 100644--- a/ministack/services/apigateway_v1.py+++ b/ministack/services/apigateway_v1.py@@ -84,7 +84,6 @@ Custom domains:data-plane requests by custom domain host today."""-import asyncioimport base64import datetimeimport json@@ -98,6 +97,7 @@ import urllib.requestimport yamlfrom ministack.core.arn import ArnParseError, parse_arn+from ministack.core.concurrency import run_reentrantfrom ministack.core.responses import (AccountRegionScopedDict,AccountScopedDict,@@ -719,7 +719,8 @@ async def _call_lambda_raw(function_ref, event, *, account_id=None, region=None)return None, f"Lambda function '{label}' not found"exec_record = lambda_svc._execution_record_for_config(func_data, func_config)- result = await asyncio.to_thread(lambda_svc._execute_function_with_config_scope, exec_record, event)+ result = await run_reentrant(lambda_svc._execute_function_with_config_scope, exec_record, event,+ thread_name="ministack-apigw-invoke")return result, None@@ -1395,8 +1396,9 @@ async def _invoke_authorizer_lambda(authorizer, event, account_id, region):if func_data is None or func_config is None:return Noneexec_record = lambda_svc._execution_record_for_config(func_data, func_config)- return await asyncio.to_thread(- lambda_svc._execute_function_with_config_scope, exec_record, event+ return await run_reentrant(+ lambda_svc._execute_function_with_config_scope, exec_record, event,+ thread_name="ministack-apigw-authorizer",)diff --git a/ministack/services/appsync_events.py b/ministack/services/appsync_events.pyindex 0dab9cd6..e6ade3b8 100644--- a/ministack/services/appsync_events.py+++ b/ministack/services/appsync_events.py@@ -26,6 +26,7 @@ import osimport reimport time+from ministack.core.concurrency import run_reentrantfrom ministack.core.persistence import load_statefrom ministack.core.responses import (AccountRegionScopedDict,@@ -545,7 +546,7 @@ async def _authorize_event_op(if not token:return False, f"{operation} rejected: no Authorization token"namespace = _channel_namespace_for(channel) if channel else None- ok, _ctx = await asyncio.to_thread(+ ok, _ctx = await run_reentrant(_events_authorizer_invoke,lambda_arn,api_id,@@ -1145,7 +1146,7 @@ async def _handle_websocket_in_api_scope():returnelse:- ok, connect_rctx = await asyncio.to_thread(+ ok, connect_rctx = await run_reentrant(_events_authorizer_invoke,lambda_auth_arn,api_id,diff --git a/ministack/services/cloudformation/stacks.py b/ministack/services/cloudformation/stacks.pyindex 63ee6c6a..7e123fbd 100644--- a/ministack/services/cloudformation/stacks.py+++ b/ministack/services/cloudformation/stacks.py@@ -4,18 +4,15 @@ CloudFormation stacks — async stack lifecycle (deploy, delete, update, diff).import asyncioimport copy-import jsonimport logging-import timefrom contextlib import contextmanager-from ministack.core.responses import get_account_id, get_region, new_uuid, now_iso, set_request_region+from ministack.core.concurrency import run_reentrant+from ministack.core.responses import get_region, new_uuid, now_iso, set_request_regionfrom .engine import (_NO_VALUE,_evaluate_conditions,- _parse_template,- _resolve_parameters,_resolve_refs,_topological_sort,)@@ -162,7 +159,7 @@ async def _deploy_stack_async(stack_name: str, stack_id: str, template: dict,old_props = prev_resource.get("Properties", {})old_attrs = prev_resource.get("Attributes", {})if _is_custom_resource(resource_type):- physical_id, attrs = await asyncio.to_thread(+ physical_id, attrs = await run_reentrant(_update_resource, resource_type, old_pid, old_props,resolved_props, stack_name, logical_id, old_attrs)@@ -173,7 +170,7 @@ async def _deploy_stack_async(stack_name: str, stack_id: str, template: dict,)else:if _is_custom_resource(resource_type):- physical_id, attrs = await asyncio.to_thread(+ physical_id, attrs = await run_reentrant(_provision_resource, resource_type, logical_id, resolved_props, stack_name)else:@@ -213,7 +210,7 @@ async def _deploy_stack_async(stack_name: str, stack_id: str, template: dict,old_props = old_res.get("Properties", {})try:if _is_custom_resource(rtype):- await asyncio.to_thread(+ await run_reentrant(_delete_resource, rtype, pid, old_props,stack_name, logical_id)@@ -247,7 +244,7 @@ async def _deploy_stack_async(stack_name: str, stack_id: str, template: dict,res_props = res.get("Properties", {})try:if _is_custom_resource(rtype):- await asyncio.to_thread(+ await run_reentrant(_delete_resource, rtype, pid, res_props,stack_name, logical_id)@@ -355,7 +352,7 @@ async def _delete_stack_async(stack_name: str, stack_id: str):"DELETE_IN_PROGRESS", physical_id=pid)try:if _is_custom_resource(rtype):- await asyncio.to_thread(+ await run_reentrant(_delete_resource, rtype, pid, res_props,stack_name, logical_id)diff --git a/ministack/services/codebuild.py b/ministack/services/codebuild.pyindex 4f835707..712b5044 100644--- a/ministack/services/codebuild.py+++ b/ministack/services/codebuild.py@@ -21,8 +21,10 @@ import reimport threadingimport time+from ministack.core import container_reaperfrom ministack.core.arn import ArnParseError, is_arn, parse_arn-from ministack.core.persistence import PERSIST_STATE, load_state+from ministack.core.concurrency import run_reentrant+from ministack.core.persistence import load_statefrom ministack.core.responses import (AccountRegionScopedDict,error_response_json,@@ -34,6 +36,11 @@ from ministack.core.responses import (logger = logging.getLogger("codebuild")+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a+# slow or wedged daemon into a minutes-long stall on a request path.+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))++REGION = os.environ.get("MINISTACK_REGION", "us-east-1")# ---------------------------------------------------------------------------@@ -218,7 +225,7 @@ def _get_docker():if _docker is None:try:import docker- _docker = docker.from_env()+ _docker = docker.from_env(timeout=_DOCKER_TIMEOUT)except Exception:logger.exception("Docker unavailable; builds stay metadata-only")_docker = False@@ -402,7 +409,7 @@ def _execute_build(build_id, project):"/var/run/docker.sock": {"bind": "/var/run/docker.sock", "mode": "rw"},env_dir: {"bind": "/LocalBuild/envFile", "mode": "ro"},},- labels={"ministack": "codebuild", "ministack.codebuild.build": build_id},+ labels=container_reaper.own_labels("codebuild", **{"ministack.codebuild.build": build_id}),)except Exception:logger.exception("Failed to start build %s", build_id)@@ -423,7 +430,7 @@ def _execute_build(build_id, project):timed_out.set()logger.error("Build %s exceeded timeoutInMinutes; stopping it", build_id)try:- container.remove(force=True)+ container.remove(force=True, v=True)except Exception:logger.exception("Could not stop the timed-out build %s", build_id)@@ -476,7 +483,7 @@ def _execute_build(build_id, project):"phases in this format", build_id, AGENT_IMAGE)try:- container.remove(force=True)+ container.remove(force=True, v=True)except Exception:pass@@ -485,7 +492,7 @@ def _execute_build(build_id, project):# Request dispatcher# ----------------------------------------------------------------------------async def handle_request(method, path, headers, body, query_params):+def _handle_request_sync(method, path, headers, body, query_params):target = headers.get("x-amz-target", "")action = target.split(".")[-1] if "." in target else ""@@ -690,7 +697,7 @@ def _stop_build(data):container = _container_for_build(bid) if EXECUTE_BUILDS else Noneif container:try:- container.remove(force=True)+ container.remove(force=True, v=True)except Exception:logger.exception("Could not stop the build container for %s", bid)@@ -732,3 +739,20 @@ def _batch_delete_builds(data):else:not_deleted.append(bid)return json_response({"buildsDeleted": deleted, "buildsNotDeleted": not_deleted})+++async def handle_request(method, path, headers, body, query_params):+ """Dispatch off the event loop.++ Request paths here reach the Docker daemon (container create/start/stop/+ inspect), which blocks for as long as the daemon takes. Measured on ECS: a+ cached-image container start held the loop for 7.3s, during which the health+ endpoint — the cheapest request in the process — could not be served.++ Uses run_reentrant, not the shared pool: containers started here are handed+ an endpoint pointing back at MiniStack, so one calls in while this+ dispatch is still running. A bounded pool would queue that nested request+ behind the call waiting on it.+ """+ return await run_reentrant(+ _handle_request_sync, method, path, headers, body, query_params, thread_name="ministack-codebuild-dispatch")diff --git a/ministack/services/cognito.py b/ministack/services/cognito.pyindex 9c2fec13..731a3a9a 100644--- a/ministack/services/cognito.py+++ b/ministack/services/cognito.py@@ -54,7 +54,6 @@ Wire protocol:Routing is handled in app.py via two separate SERVICE_HANDLERS entries."""-import asyncioimport base64import copyimport hashlib@@ -75,7 +74,8 @@ from xml.etree.ElementTree import tostring as xml_tostringfrom defusedxml.ElementTree import fromstring as safe_xml_parsefrom ministack.core.arn import ArnParseError, parse_arn-from ministack.core.persistence import PERSIST_STATE, load_state+from ministack.core.concurrency import run_reentrant+from ministack.core.persistence import load_statefrom ministack.core.responses import (AccountRegionScopedDict,AccountScopedDict,@@ -1919,7 +1919,8 @@ async def _dispatch_idp(action: str, data: dict):# back into ministack over HTTP) don't deadlock against a blocked event# loop. The Lambda response path needs the loop free to accept new# requests while _execute_function is running.- return await asyncio.to_thread(_run_idp_handler, handler, action, data)+ return await run_reentrant(_run_idp_handler, handler, action, data,+ thread_name="ministack-cognito-trigger")# ---------------------------------------------------------------------------@@ -1950,7 +1951,8 @@ async def _dispatch_identity(action: str, data: dict):handler = handlers.get(action)if not handler:return error_response_json("InvalidAction", f"Unknown Cognito Identity action: {action}", 400)- return await asyncio.to_thread(_run_identity_handler, handler, action, data)+ return await run_reentrant(_run_identity_handler, handler, action, data,+ thread_name="ministack-cognito-trigger")# ===========================================================================diff --git a/ministack/services/dsql.py b/ministack/services/dsql.pyindex d89e8735..e8cd7583 100644--- a/ministack/services/dsql.py+++ b/ministack/services/dsql.py@@ -21,7 +21,8 @@ import stringimport threadingimport time-from ministack.core import pgproxy+from ministack.core import container_reaper, pgproxy+from ministack.core.concurrency import run_offloopfrom ministack.core.persistence import load_statefrom ministack.core.responses import (AccountScopedDict,@@ -245,7 +246,7 @@ def _run_backend_container(identifier):},ports={"5432/tcp": None}, # docker-assigned host portname=name,- labels={"ministack": "dsql", "cluster_id": identifier},+ labels=container_reaper.own_labels("dsql", cluster_id=identifier),)if ms_network:container_kwargs["network"] = ms_network@@ -306,6 +307,22 @@ def _remove_container(identifier):logger.debug("DSQL: container removal for %s: %s", identifier, e)+def _serving_loop():+ """The loop serving HTTP, or None outside a running server.++ ``_main_loop`` is only populated by the ``start_restored_proxies`` lifespan+ hook, which is skipped when the dsql module was not already loaded at boot —+ i.e. on every fresh server whose first dsql call is CreateCluster. The app+ captures the loop unconditionally, so fall back to that.+ """+ try:+ from ministack import app++ return app._MAIN_LOOP+ except Exception:+ return None++async def _start_backend(identifier, cluster):"""Background task: container + proxy, then flip the cluster ACTIVE.@@ -457,10 +474,21 @@ def _create_cluster(data):_tags[arn] = dict(data["tags"])if has_backend:+ # This handler runs on a worker thread (dispatch is off the event loop so+ # Docker work cannot freeze the server), so there is no *running* loop+ # here to create_task on. Schedule onto the serving loop instead — the+ # same pattern _start_backend's respawn path already uses. Falling back+ # to get_running_loop covers direct/in-process callers that do have one.+ loop = _main_loop or _serving_loop()try:- asyncio.get_running_loop().create_task(_start_backend(identifier, cluster))+ if loop is not None and loop.is_running():+ asyncio.run_coroutine_threadsafe(+ _start_backend(identifier, cluster), loop)+ else:+ asyncio.get_running_loop().create_task(+ _start_backend(identifier, cluster))except RuntimeError:- # No event loop (direct/module-level call) — metadata-only.+ # No event loop at all (module-level call) — metadata-only.cluster["status"] = "ACTIVE"cluster["_has_backend"] = False@@ -742,7 +770,7 @@ def _list_tags_for_resource(arn):# ----------------------------------------------------------------------------async def handle_request(method, path, headers, body, query_params):+def _handle_request_sync(method, path, headers, body, query_params):try:data = json.loads(body) if body else {}if not isinstance(data, dict):@@ -905,3 +933,18 @@ try:restore_state(_restored)except Exception:logging.getLogger(__name__).exception("Failed to restore persisted state; continuing with fresh store")+++async def handle_request(method, path, headers, body, query_params):+ """Dispatch off the event loop.++ Request paths here reach the Docker daemon (container create/start/stop/+ inspect), which blocks for as long as the daemon takes. Measured on ECS: a+ cached-image container start held the loop for 7.3s, during which the health+ endpoint — the cheapest request in the process — could not be served.++ Uses the shared pool: the containers started here (database / cache / search+ engines) never call back into MiniStack, so this cannot re-enter.+ """+ return await run_offloop(+ _handle_request_sync, method, path, headers, body, query_params)diff --git a/ministack/services/ecs.py b/ministack/services/ecs.pyindex 373a2b78..209691fc 100644--- a/ministack/services/ecs.py+++ b/ministack/services/ecs.py@@ -31,7 +31,9 @@ import secretsimport threadingimport time+from ministack.core import container_reaperfrom ministack.core.arn import ArnParseError, parse_arn+from ministack.core.concurrency import run_reentrantfrom ministack.core.persistence import load_statefrom ministack.core.responses import (AccountRegionScopedDict,@@ -47,6 +49,11 @@ from ministack.services import ecs_metadata, secretsmanagerlogger = logging.getLogger("ecs")+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a+# slow or wedged daemon into a minutes-long stall on a request path.+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))++REGION = os.environ.get("MINISTACK_REGION", "us-east-1")_MINISTACK_HOST = os.environ.get("MINISTACK_HOST", "localhost")_GATEWAY_PORT = os.environ.get("GATEWAY_PORT", "4566")@@ -80,18 +87,29 @@ _ecs_reaper_started = False_ecs_reaper_lock = threading.Lock()+def _live_container_ids():+ """Container ids still owned by a live task.++ Registered with ``core.container_reaper``, which reclaims ECS containers+ that no task references any more. This replaces a service-local reaper that+ removed every exited/dead/created ECS container regardless of ownership —+ correct in practice, but a second implementation of the same rule, and one+ that could race a task between `created` and `running`.+ """+ ids = set()+ for _key, task in _tasks.all_items():+ for cid in task.get("_docker_ids") or []:+ if cid:+ ids.add(cid)+ return ids+++container_reaper.register_live_ids("ecs", _live_container_ids)++def _reap_exited_containers():- docker_client = _get_docker()- if not docker_client:- return- # all=True includes exited containers; reset() uses list() (running only),- # so this complements — and overlaps harmlessly — with the reset path.- for c in docker_client.containers.list(all=True, filters={"label": "ministack=ecs"}):- if c.status in ("exited", "dead", "created"):- try:- c.remove(v=True, force=True)- except Exception:- pass+ """Kept as a no-op shim: reclamation is owned by core.container_reaper."""+ returndef _ensure_ecs_reaper_thread():@@ -236,7 +254,7 @@ def _get_docker():if _docker is None:try:import docker- _docker = docker.from_env()+ _docker = docker.from_env(timeout=_DOCKER_TIMEOUT)except Exception:passreturn _docker@@ -265,7 +283,7 @@ def _ts_to_epoch(value):if not isinstance(value, str):return valuetry:- from datetime import datetime, timezone+ from datetime import datetimereturn int(datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp())except (ValueError, TypeError):return value@@ -309,13 +327,26 @@ async def handle_request(method, path, headers, body, query_params):target = headers.get("x-amz-target", "") or headers.get("X-Amz-Target", "")if target:action = target.split(".")[-1]- return _finalize_ecs_response(_dispatch_action(action, data))+ # Docker work (RunTask/StopTask/service updates) blocks for as long as+ # the daemon takes: a cached nginx start already holds the loop >7s,+ # measured, during which no service in the process can be served.+ #+ # run_reentrant, not the shared pool: every task container is handed+ # AWS_ENDPOINT_URL pointing back here (see _task_env), so a starting+ # container calls into MiniStack while this call is still in flight. A+ # bounded pool — or a per-service lock — would put that nested request+ # behind the call waiting on it.+ return _finalize_ecs_response(+ await run_reentrant(_dispatch_action, action, data,+ thread_name="ministack-ecs-dispatch"))parts = [p for p in path.strip("/").split("/") if p]if not parts:return error_response_json("ClientException", "Missing path", 400)- return _finalize_ecs_response(_dispatch_path(method, parts, data))+ return _finalize_ecs_response(+ await run_reentrant(_dispatch_path, method, parts, data,+ thread_name="ministack-ecs-dispatch"))def _dispatch_action(action, data):@@ -1069,7 +1100,7 @@ def _build_run_kwargs(cdef, td, env, port_bindings, ecs_network,ports=port_bindings or None,name=f"ministack-ecs-{task_id[:8]}-{cdef['name']}",labels={- "ministack": "ecs",+ **container_reaper.own_labels("ecs"),"com.amazonaws.ecs.cluster": cluster_arn,"com.amazonaws.ecs.container-name": cdef["name"],"com.amazonaws.ecs.task-arn": task_arn,@@ -2137,13 +2168,13 @@ def reset():# (echo, wget probes) leave Exited containers behind that otherwise# only the background reaper would remove.try:- for c in docker_client.containers.list(all=True, filters={"label": "ministack=ecs"}):- try:- if c.status == "running":- c.stop(timeout=2)- c.remove(v=True, force=True)- except Exception:- pass+ from ministack.core import container_reaper+ # Concurrently: reset holds the global reset lock, and stopping a+ # run's worth of tasks one at a time is tens of seconds during which+ # every other request is blocked.+ container_reaper.drop_containers(+ docker_client.containers.list(all=True, filters={"label": "ministack=ecs"}),+ stop_timeout=2, force=True)except Exception:pass_clusters.clear()diff --git a/ministack/services/eks.py b/ministack/services/eks.pyindex f8728256..90583428 100644--- a/ministack/services/eks.py+++ b/ministack/services/eks.py@@ -24,21 +24,26 @@ import threadingimport timeimport urllib.parse+from ministack.core import container_reaperfrom ministack.core.arn import ArnParseError, parse_arn+from ministack.core.concurrency import run_reentrantfrom ministack.core.persistence import load_statefrom ministack.core.responses import (AccountRegionScopedDict,AccountScopedDict,apply_image_prefix,- error_response_json,get_account_id,get_region,- json_response,new_uuid,)logger = logging.getLogger("eks")+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a+# slow or wedged daemon into a minutes-long stall on a request path.+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))++REGION = os.environ.get("MINISTACK_REGION", "us-east-1")_MINISTACK_HOST = os.environ.get("MINISTACK_HOST", "localhost")EKS_K3S_IMAGE = os.environ.get("EKS_K3S_IMAGE", "rancher/k3s:v1.31.4-k3s1")@@ -297,7 +302,7 @@ def _get_docker():if not _docker_available:return Nonetry:- return docker_lib.from_env()+ return docker_lib.from_env(timeout=_DOCKER_TIMEOUT)except Exception:return None@@ -418,7 +423,7 @@ def _k3s_run_kwargs(devices=["/dev/fuse"],ports={"6443/tcp": port},name=f"ministack-eks-{region}-{name}",- labels={"ministack": "eks", "cluster_name": name, "region": region},+ labels=container_reaper.own_labels("eks", cluster_name=name, region=region),environment={"K3S_KUBECONFIG_MODE": "644"},volumes={"/lib/modules": {"bind": "/lib/modules", "mode": "ro"}},tmpfs={"/run": "", "/var/run": "", "/tmp": ""},@@ -1454,7 +1459,7 @@ def _sanitize(cluster):# Request Router# ----------------------------------------------------------------------------async def handle_request(method, path, headers, body_bytes, query_params):+def _handle_request_sync(method, path, headers, body_bytes, query_params):try:body = json.loads(body_bytes) if body_bytes else {}except json.JSONDecodeError:@@ -1628,3 +1633,38 @@ async def handle_request(method, path, headers, body_bytes, query_params):return _untag_resource(arn, query)return _error(400, "InvalidRequestException", f"No route for {method} {path}")+++async def handle_request(method, path, headers, body, query_params):+ """Dispatch off the event loop.++ Request paths here reach the Docker daemon (container create/start/stop/+ inspect), which blocks for as long as the daemon takes. Measured on ECS: a+ cached-image container start held the loop for 7.3s, during which the health+ endpoint — the cheapest request in the process — could not be served.++ Uses run_reentrant, not the shared pool: containers started here are handed+ an endpoint pointing back at MiniStack, so one calls in while this+ dispatch is still running. A bounded pool would queue that nested request+ behind the call waiting on it.+ """+ return await run_reentrant(+ _handle_request_sync, method, path, headers, body, query_params, thread_name="ministack-eks-dispatch")+++def _live_container_ids():+ """Container ids still owned by a live EKS cluster.++ EKS records its k3s container under ``_docker_id`` (not the+ ``_docker_container_id`` the other services use), which is why this was+ missed first time round and its exited containers were never reclaimed.+ """+ ids = set()+ for _key, cluster in _clusters.all_items():+ cid = cluster.get("_docker_id")+ if cid:+ ids.add(cid)+ return ids+++container_reaper.register_live_ids("eks", _live_container_ids)diff --git a/ministack/services/elasticache.py b/ministack/services/elasticache.pyindex 5909700b..d025936a 100644--- a/ministack/services/elasticache.py+++ b/ministack/services/elasticache.py@@ -20,13 +20,16 @@ When Docker is available, CreateCacheCluster spins up a real Redis/Valkey/Memcaccontainer. Otherwise returns localhost:6379 (assumes Redis sidecar in docker-compose)."""+import contextvarsimport copyimport loggingimport osimport timefrom urllib.parse import parse_qs+from ministack.core import container_reaperfrom ministack.core.arn import ArnParseError, parse_arn+from ministack.core.concurrency import resource_lock, run_offloop, spawn_backgroundfrom ministack.core.persistence import load_statefrom ministack.core.responses import (AccountRegionScopedDict,@@ -257,7 +260,7 @@ def _ensure_live_containers_locked():name=f"ministack-elasticache-{account_id}-{region}-{name}",engine=engine, engine_version=version,labels={- "ministack": "elasticache",+ **container_reaper.own_labels("elasticache"),"cluster_id": name,"account_id": account_id,"region": region,@@ -289,7 +292,7 @@ def _ensure_live_containers_locked():name=f"ministack-elasticache-rg-{account_id}-{region}-{rg_id}-{ng_id}",engine=engine, engine_version=engine_version,labels={- "ministack": "elasticache", "rg_id": rg_id,+ **container_reaper.own_labels("elasticache"), "rg_id": rg_id,"node_group": ng_id, "account_id": account_id,"region": region,},@@ -399,12 +402,27 @@ def _validate_modify_replication_group_request(p):return None+# Actions whose implementation talks to the Docker daemon (container run/get/+# stop). Kept explicit so a new action is off-loop by decision, not by accident.+_DOCKER_BACKED_ACTIONS = {+ "CreateCacheCluster", "DeleteCacheCluster", "ModifyCacheCluster",+ "RebootCacheCluster", "CreateReplicationGroup", "DeleteReplicationGroup",+ "ModifyReplicationGroup", "IncreaseReplicaCount", "DecreaseReplicaCount",+ "DescribeCacheClusters", "DescribeReplicationGroups",+}+++# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a+# slow or wedged daemon into a minutes-long stall on a request path.+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))++def _get_docker():global _dockerif _docker is None:try:import docker- _docker = docker.from_env()+ _docker = docker.from_env(timeout=_DOCKER_TIMEOUT)except Exception:passreturn _docker@@ -429,6 +447,20 @@ def _engine_image_and_port(engine, engine_version):return apply_image_prefix(f"memcached:{engine_version}-alpine"), 11211+def _image_is_local(docker_client, image: str) -> bool:+ """True when the image is already pulled.++ ``containers.run`` auto-pulls on ImageNotFound, so calling it with a cold+ cache blocks the request for as long as the pull takes. AWS returns+ ``creating`` in milliseconds and provisions asynchronously.+ """+ try:+ docker_client.images.get(image)+ return True+ except Exception:+ return False++def _spawn_redis_container(name, engine, engine_version, labels):"""Start a redis/valkey/memcached container.@@ -519,7 +551,7 @@ def _spawn_redis_cluster_node(name, engine, engine_version, labels):if not container_ip:try:container.stop(timeout=2)- container.remove()+ container.remove(v=True)except Exception:passlogger.warning("ElastiCache: cluster node %s has no IP on network %s", name, DOCKER_NETWORK)@@ -591,7 +623,7 @@ def _teardown_containers(docker_client, container_ids):try:c = docker_client.containers.get(cid)c.stop(timeout=2)- c.remove()+ c.remove(v=True)except Exception as e:logger.warning("ElastiCache: cleanup failed for %s: %s", cid, e)@@ -621,7 +653,7 @@ def _build_real_cluster_rg(rg_id, engine, engine_version, num_node_groups, replicontainer_ids = []common_labels = {- "ministack": "elasticache",+ **container_reaper.own_labels("elasticache"),"rg_id": rg_id,"account_id": account_id,"region": region,@@ -866,6 +898,11 @@ async def handle_request(method, path, headers, body, query_params):handler = handlers.get(action)if not handler:return _error("InvalidAction", f"Unknown ElastiCache action: {action}", 400)+ # Docker-backed actions block on the daemon — a cold image pull is minutes —+ # and would hold the event loop for every other service. Shared pool, not a+ # dedicated thread: a Redis/Valkey container never calls back into MiniStack.+ if action in _DOCKER_BACKED_ACTIONS:+ return await run_offloop(handler, params)return handler(params)@@ -884,17 +921,27 @@ def _create_cache_cluster(p):arn = _arn_cluster(cluster_id)account_id = get_account_id()region = get_region()- endpoint_host, endpoint_port, docker_container_id = _spawn_redis_container(+ _spawn_kwargs = dict(name=f"ministack-elasticache-{account_id}-{region}-{cluster_id}",engine=engine,engine_version=engine_version,labels={- "ministack": "elasticache",+ **container_reaper.own_labels("elasticache"),"cluster_id": cluster_id,"account_id": account_id,"region": region,},)+ _image, _port = _engine_image_and_port(engine, engine_version)+ _dc = _get_docker()+ _deferred = bool(_dc) and not _image_is_local(_dc, _image)+ if _deferred:+ # Cold image cache: pulling inside CreateCacheCluster would block the+ # API for as long as the pull takes. AWS returns `creating` at once, so+ # spawn in the background and flip to `available` when it lands.+ endpoint_host, endpoint_port, docker_container_id = "localhost", _port, None+ else:+ endpoint_host, endpoint_port, docker_container_id = _spawn_redis_container(**_spawn_kwargs)subnet_group = _p(p, "CacheSubnetGroupName") or "default"param_group_name = _p(p, "CacheParameterGroupName") or _default_param_group_for_engine(engine, engine_version)@@ -902,7 +949,7 @@ def _create_cache_cluster(p):_clusters[cluster_id] = {"CacheClusterId": cluster_id,"CacheClusterArn": arn,- "CacheClusterStatus": "available",+ "CacheClusterStatus": "creating" if _deferred else "available","Engine": engine,"EngineVersion": engine_version,"CacheNodeType": node_type,@@ -935,29 +982,103 @@ def _create_cache_cluster(p):"_endpoint": {"Address": endpoint_host, "Port": endpoint_port},}+ if _deferred:+ # Pull + start off the request path; the cluster reports `creating`+ # until this lands, exactly as AWS does.+ created_record = _clusters[cluster_id]++ def _finish_cluster_start():+ try:+ host, port, cid = _spawn_redis_container(**_spawn_kwargs)+ except Exception:+ with resource_lock("elasticache", cluster_id):+ rec = _clusters.get(cluster_id)+ if rec is created_record:+ rec["CacheClusterStatus"] = "create-failed"+ return+ # Publish under the lock so a DeleteCacheCluster cannot land between+ # the "is it still there" check and the write. Identity, not+ # truthiness: a same-named cluster created after a delete is a+ # different resource and must not adopt this container.+ with resource_lock("elasticache", cluster_id):+ rec = _clusters.get(cluster_id)+ orphaned = rec is not created_record or rec.get("_deleting")+ if not orphaned:+ rec["_docker_container_id"] = cid+ _publish_endpoint(rec, host, port)+ rec["CacheClusterStatus"] = "available"+ if orphaned:+ # Deleted while the image was pulling. Nothing references this+ # container and the periodic reaper only touches created / dead+ # / exited, never a *running* one — so it would run forever.+ # Torn down outside the lock: Docker calls never hold it.+ _teardown_containers(_get_docker(), [cid])++ spawn_background(contextvars.copy_context().run, _finish_cluster_start,+ thread_name=f"ministack-elasticache-start-{cluster_id}")+_tags[arn] = _extract_tags(p)_record_event(cluster_id, "cache-cluster", "Cache cluster created")return _xml_cluster_response("CreateCacheClusterResponse", "CreateCacheClusterResult", _clusters[cluster_id])+def _publish_endpoint(rec: dict, host: str, port: int) -> None:+ """Write the cluster's endpoint everywhere a reader looks for it.++ The deferred-start bug was two writers disagreeing about the key: create+ recorded ``_endpoint`` and ``CacheNodes[].Endpoint`` (with the *engine+ default* port, since no container existed yet), while the finisher updated+ only ``ConfigurationEndpoint``. Every Describe path reads the first two, so a+ cold-image create reported ``localhost:6379`` for the rest of its life while+ the container was on the allocated host port. One writer, so they cannot+ drift apart again.+ """+ endpoint = {"Address": host, "Port": port}+ rec["_endpoint"] = dict(endpoint)+ for node in rec.get("CacheNodes") or []:+ node["Endpoint"] = dict(endpoint)+ if rec.get("ConfigurationEndpoint") is not None or "ConfigurationEndpoint" in rec:+ rec["ConfigurationEndpoint"] = dict(endpoint)++def _delete_cache_cluster(p):cluster_id = _p(p, "CacheClusterId")- cluster = _clusters.get(cluster_id)- if not cluster:- return _error("CacheClusterNotFound", f"Cluster {cluster_id} not found", 404)+ # Claim the record under the lock and leave a tombstone, so a deferred+ # start still in flight sees the delete and reclaims what it started+ # instead of publishing a container nobody references.+ with resource_lock("elasticache", cluster_id):+ cluster = _clusters.get(cluster_id)+ if not cluster:+ return _error("CacheClusterNotFound", f"Cluster {cluster_id} not found", 404)+ cluster["_deleting"] = True+ cluster["CacheClusterStatus"] = "deleting"+ container_id = cluster.get("_docker_container_id")+ del _clusters[cluster_id]docker_client = _get_docker()- if docker_client and cluster.get("_docker_container_id"):- try:- container = docker_client.containers.get(cluster["_docker_container_id"])- container.stop(timeout=5)- container.remove()- except Exception as e:- logger.warning("ElastiCache: failed to remove container for %s: %s", cluster_id, e)-- cluster["CacheClusterStatus"] = "deleting"- del _clusters[cluster_id]+ if docker_client:+ # By id *or* by the deterministic name. The id alone is not enough: a+ # delete can arrive before the deferred start has published one, and+ # then nothing would ever reclaim the container. Two independent+ # handles, so a leak needs both to miss.+ for locate in (+ (lambda: docker_client.containers.get(container_id)) if container_id else None,+ lambda: docker_client.containers.get(+ f"ministack-elasticache-{get_account_id()}-{get_region()}-{cluster_id}"),+ ):+ if locate is None:+ continue+ try:+ container = locate()+ except Exception:+ continue+ try:+ container.stop(timeout=5)+ container.remove(v=True)+ except Exception as e:+ logger.warning("ElastiCache: failed to remove container for %s: %s", cluster_id, e)+ break_tags.pop(cluster.get("CacheClusterArn", ""), None)_record_event(cluster_id, "cache-cluster", "Cache cluster deleted")return _xml_cluster_response("DeleteCacheClusterResponse", "DeleteCacheClusterResult", cluster)@@ -1108,7 +1229,7 @@ def _create_replication_group(p):engine=engine,engine_version=engine_version,labels={- "ministack": "elasticache",+ **container_reaper.own_labels("elasticache"),"rg_id": rg_id,"node_group": ng_id,"account_id": account_id,@@ -1258,7 +1379,7 @@ def _delete_replication_group(p):try:container = docker_client.containers.get(cid)container.stop(timeout=5)- container.remove()+ container.remove(v=True)except Exception as e:logger.warning("ElastiCache: failed to remove RG container %s for %s: %s", cid, rg_id, e)@@ -2416,3 +2537,20 @@ def reset():_pending_rg_respawn.clear()_port_counter[0] = BASE_PORTdefault_state()+++def _live_container_ids():+ ids = set()+ for _key, rec in _clusters.all_items():+ cid = rec.get("_docker_container_id")+ if cid:+ ids.add(cid)+ for cid2 in rec.get("_docker_container_ids") or []:+ ids.add(cid2)+ for _key, rg in _replication_groups.all_items():+ for cid2 in rg.get("_docker_container_ids") or []:+ ids.add(cid2)+ return ids+++container_reaper.register_live_ids("elasticache", _live_container_ids)diff --git a/ministack/services/firehose.py b/ministack/services/firehose.pyindex 19e130fb..00299b05 100644--- a/ministack/services/firehose.py+++ b/ministack/services/firehose.py@@ -25,7 +25,8 @@ import threadingimport timefrom ministack.core.arn import ArnParseError, parse_arn-from ministack.core.persistence import PERSIST_STATE, load_state+from ministack.core.concurrency import spawn_background+from ministack.core.persistence import load_statefrom ministack.core.responses import (AccountRegionScopedDict,error_response_json,@@ -521,7 +522,7 @@ def _deliver_to_iceberg(stream: dict, dest: dict, records: list):if not groups:return- def _run():+ def _deliver_iceberg_groups():for (wh, db, table, keys), group in groups.items():try:_iceberg_write_group(wh, db, table, list(keys), group)@@ -529,11 +530,14 @@ def _deliver_to_iceberg(stream: dict, dest: dict, records: list):logger.warning("Firehose %s: Iceberg delivery to %s.%s failed: %s",name, db, table, exc)+ # Inline if no thread can be started: losing delivery records silently is+ # worse than blocking this caller. (Before this ran on `spawn_background`,+ # the fallback existed for a different reason — `get_running_loop()` raising+ # off the event loop — which can no longer happen.)try:- loop = asyncio.get_running_loop()- loop.run_in_executor(None, _run)+ spawn_background(_deliver_iceberg_groups, thread_name="ministack-firehose-iceberg")except RuntimeError:- _run()+ _deliver_iceberg_groups()def _record_id() -> str:diff --git a/ministack/services/glue.py b/ministack/services/glue.pyindex 111b19a3..a62402f7 100644--- a/ministack/services/glue.py+++ b/ministack/services/glue.py@@ -24,8 +24,10 @@ import threadingimport timefrom urllib.parse import unquote+from ministack.core import container_reaperfrom ministack.core.arn import ArnParseError, parse_arn-from ministack.core.persistence import PERSIST_STATE, load_state+from ministack.core.concurrency import run_reentrant+from ministack.core.persistence import load_statefrom ministack.core.responses import (AccountRegionScopedDict,AccountScopedDict,@@ -38,6 +40,11 @@ from ministack.core.responses import (logger = logging.getLogger("glue")+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a+# slow or wedged daemon into a minutes-long stall on a request path.+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))++REGION = os.environ.get("MINISTACK_REGION", "us-east-1")CRAWLER_RUN_SECONDS = int(os.environ.get("GLUE_CRAWLER_RUN_SECONDS", "5"))S3_DATA_DIR = os.environ.get("S3_DATA_DIR", "/tmp/ministack-data/s3")@@ -60,7 +67,7 @@ def _get_docker():if _docker is None:try:import docker- _docker = docker.from_env()+ _docker = docker.from_env(timeout=_DOCKER_TIMEOUT)except Exception:passreturn _docker@@ -259,7 +266,7 @@ def _invalid_resource_arn(arn):return error_response_json("InvalidInputException", f"Invalid Glue resource ARN: {arn}", 400)-async def handle_request(method, path, headers, body, query_params):+def _handle_request_sync(method, path, headers, body, query_params):# Glue's Iceberg REST catalog data plane. On real AWS this is# `glue.<region>.amazonaws.com/iceberg` — same service, same SigV4# signing name (`glue`), different protocol (Iceberg REST OpenAPI@@ -1448,7 +1455,7 @@ def _execute_spark_docker(run, job, job_name, args, script_path, docker_client):# Remove stale container with same nametry:existing = docker_client.containers.get(container_name)- existing.remove(force=True)+ existing.remove(force=True, v=True)except Exception:pass@@ -1531,7 +1538,7 @@ def _execute_spark_docker(run, job, job_name, args, script_path, docker_client):"command": cmd,"environment": container_env,"detach": True,- "labels": {"ministack": "glue", "job_name": job_name},+ "labels": container_reaper.own_labels("glue", job_name=job_name),}if ms_network:@@ -1561,7 +1568,7 @@ def _execute_spark_docker(run, job, job_name, args, script_path, docker_client):run["JobRunState"] = "FAILED"run["ErrorMessage"] = f"Docker container start failed: {e}"[:2000]try:- container.remove(force=True)+ container.remove(force=True, v=True)except Exception:passreturn@@ -1585,7 +1592,7 @@ def _execute_spark_docker(run, job, job_name, args, script_path, docker_client):logger.warning("Glue: Spark container for %s error: %s", job_name, e)finally:try:- container.remove(force=True)+ container.remove(force=True, v=True)except Exception:pass@@ -2094,3 +2101,20 @@ def reset():_user_defined_functions.clear()_table_column_statistics.clear()_partition_column_statistics.clear()+++async def handle_request(method, path, headers, body, query_params):+ """Dispatch off the event loop.++ Request paths here reach the Docker daemon (container create/start/stop/+ inspect), which blocks for as long as the daemon takes. Measured on ECS: a+ cached-image container start held the loop for 7.3s, during which the health+ endpoint — the cheapest request in the process — could not be served.++ Uses run_reentrant, not the shared pool: containers started here are handed+ an endpoint pointing back at MiniStack, so one calls in while this+ dispatch is still running. A bounded pool would queue that nested request+ behind the call waiting on it.+ """+ return await run_reentrant(+ _handle_request_sync, method, path, headers, body, query_params, thread_name="ministack-glue-dispatch")diff --git a/ministack/services/lambda_svc.py b/ministack/services/lambda_svc.pyindex 9f0d95f7..56ec7e41 100644--- a/ministack/services/lambda_svc.py+++ b/ministack/services/lambda_svc.py@@ -24,7 +24,6 @@ with the event piped through stdin (safe from injection).SQS event source mappings poll the queue in a background thread."""-import asyncioimport base64import contextvarsimport copy@@ -48,16 +47,20 @@ from datetime import datetime, timezonefrom typing import Anyfrom urllib.parse import quote, unquote+from ministack.core import container_reaperfrom ministack.core.arn import ArnParseError, parse_arn+from ministack.core.concurrency import run_reentrantfrom ministack.core.lambda_runtime import (INVOKE_DEPTH_BOOTSTRAP,INVOKE_DEPTH_ENV,INVOKE_DEPTH_EVENT_KEY,INVOKE_DEPTH_HEADER,- get_or_create_worker,+ acquire_worker,invalidate_worker,+ reap_idle_workers,+ release_worker,)-from ministack.core.persistence import PERSIST_STATE, STATE_DIR, load_state+from ministack.core.persistence import STATE_DIR, load_statefrom ministack.core.responses import (_12_DIGIT_RE,AccountRegionScopedDict,@@ -74,6 +77,11 @@ from ministack.core.responses import (logger = logging.getLogger("lambda")+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a+# slow or wedged daemon into a minutes-long stall on a request path.+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))++_MINISTACK_HOST = os.environ.get("MINISTACK_HOST", "localhost")@@ -301,7 +309,7 @@ def _get_docker_client():if not _docker_available:return Nonetry:- _cached_docker_client = docker_lib.from_env()+ _cached_docker_client = docker_lib.from_env(timeout=_DOCKER_TIMEOUT)return _cached_docker_clientexcept Exception:return None@@ -2583,7 +2591,8 @@ async def _invoke(name: str, event: dict, headers: dict, path_qualifier: str | N# RequestResponse — execute in worker thread so nested SDK calls# from the Lambda process can still reach this ASGI server._t_start = time.time()- result = await asyncio.to_thread(_execute_function_with_config_scope, exec_record, event)+ result = await run_reentrant(_execute_function_with_config_scope, exec_record, event,+ thread_name="ministack-lambda-invoke")_duration_ms = (time.time() - _t_start) * 1000.0_emit_lambda_metrics(name,@@ -2763,10 +2772,73 @@ _KEEPALIVE_KILL_ON_RELEASE = os.environ.get("LAMBDA_KEEPALIVE_MS", "").strip() =# Per-function concurrency: only applied when ReservedConcurrentExecutions is# explicitly set on the function. Otherwise the function can consume the# full account pool, matching AWS.-# Account-level concurrency cap: AWS default is 1000. We default to unbounded-# locally (a laptop can't actually run 1000 Lambda containers); users who-# want AWS-exact throttling behaviour can set LAMBDA_ACCOUNT_CONCURRENCY.-_ACCOUNT_CONCURRENCY_CAP = int(os.environ.get("LAMBDA_ACCOUNT_CONCURRENCY", "0")) # 0 = unbounded+# Per-invocation memory cost of an in-flight execution, measured: a chain of+# depth 40 took the container to 871 MiB of cgroup memory, depth 20 to 464 MiB.+_INFLIGHT_MIB = 21++# Fraction of the memory budget concurrency may claim, leaving room for the+# server itself, page cache, and whatever the handlers allocate.+_INFLIGHT_BUDGET = 0.5++# AWS's own default account concurrency. Never exceed it: a MiniStack that+# throttled later than AWS would let code through that AWS rejects.+_AWS_DEFAULT_ACCOUNT_CONCURRENCY = 1000+++def _memory_budget_bytes() -> int:+ """Total memory this process may plan against — cgroup limit if capped."""+ for path in ("/sys/fs/cgroup/memory.max", # cgroup v2+ "/sys/fs/cgroup/memory/memory.limit_in_bytes"): # cgroup v1+ try:+ with open(path) as fh:+ raw = fh.read().strip()+ if raw and raw != "max":+ value = int(raw)+ # An uncapped cgroup reports a sentinel near 2**63; ignore it.+ if 0 < value < (1 << 62):+ return value+ except Exception:+ pass+ try:+ return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")+ except Exception:+ return 2 * 1024 ** 3 # unknowable: assume a small box and stay safe+++def _default_account_concurrency() -> int:+ """How many executions may be in flight before we throttle.++ AWS defaults this to 1000 and throttles past it with+ ``TooManyRequestsException``. MiniStack cannot run 1000 Lambda+ subprocesses, and the previous default here was **unbounded** — the+ reasoning being that a laptop can't do 1000 anyway. That is backwards: with+ no bound, one self-recursing handler takes the process down. It is reachable+ because ``MAX_INVOKE_DEPTH`` is enforced from a header that only botocore's+ ``before-call`` hook sets, so a handler whose nested Invoke goes over raw+ urllib or requests never increments the depth and never trips the limit.+ Measured: such a handler OOM-killed a 2 GiB container in 8 seconds, 93+ processes deep, where the depth check never fired once.++ Depth therefore cannot be the safety bound — any transport that bypasses+ botocore evades it. Concurrency has to be, because every executor passes+ through ``_acquire_execution_slot`` regardless of how the caller got here.++ Scales to the box so a big machine is not throttled at a small machine's+ limit, and never exceeds AWS's own 1000. ``LAMBDA_ACCOUNT_CONCURRENCY``+ still overrides, including ``0`` for genuinely unbounded.+ """+ raw = os.environ.get("LAMBDA_ACCOUNT_CONCURRENCY")+ if raw is not None and raw.strip() != "":+ try:+ return int(raw) # explicit wins, 0 still means unbounded+ except ValueError:+ logger.warning("LAMBDA_ACCOUNT_CONCURRENCY=%r is not an integer; using the default",+ raw)+ budget_mib = (_memory_budget_bytes() * _INFLIGHT_BUDGET) / (1024 ** 2)+ return max(16, min(_AWS_DEFAULT_ACCOUNT_CONCURRENCY, int(budget_mib // _INFLIGHT_MIB)))+++_ACCOUNT_CONCURRENCY_CAP = _default_account_concurrency() # 0 = unbounded_reaper_started = False_reaper_lock = threading.Lock()@@ -2795,7 +2867,7 @@ def _kill_pool_entry(entry: dict) -> None:except Exception:passtry:- container.remove(force=True)+ container.remove(force=True, v=True)except Exception:passtmpdir = entry.get("tmpdir")@@ -2947,6 +3019,13 @@ def _ensure_reaper_thread() -> None:_pool_evict_idle()except Exception as exc:logger.debug("Lambda pool reaper iteration error: %s", exc)+ try:+ # Local executor: surplus warm *subprocesses* left behind by+ # a concurrency burst. Without this the per-function pools+ # only grow.+ reap_idle_workers()+ except Exception as exc:+ logger.debug("Lambda worker reaper iteration error: %s", exc)threading.Thread(target=_loop, daemon=True, name="ministack-lambda-reaper").start()_reaper_started = True@@ -3706,7 +3785,7 @@ def _spawn_lambda_container(config: dict, code_zip: bytes | None):"ports": {"8080/tcp": None},"detach": True,"stdin_open": False,- "labels": {"ministack": "lambda"},+ "labels": container_reaper.own_labels("lambda"),}if package_type == "Image":# User image brings its own entrypoint. ImageConfig can override.@@ -4039,6 +4118,80 @@ def _emit_lambda_logs(func: dict, request_id: str, log_text: str,logger.debug("CW Logs emit failed for %s: %s", func.get("config", {}).get("FunctionName"), exc)+# ---------------------------------------------------------------------------+# Concurrency accounting — executor-agnostic+# ---------------------------------------------------------------------------+#+# AWS model: concurrent invocations of a function each occupy one unit of+# concurrency. ReservedConcurrentExecutions caps a single function; the account+# limit caps every function together. Neither queues — at the limit AWS returns+# ConcurrentInvocationLimitExceeded (429).+#+# This lives at _execute_function rather than in any executor's worker pool+# because MiniStack has six executors (warm subprocess, docker RIE, provided+# runtime, proxy, one-off subprocess, and the ESM/WebSocket path) and only some+# of them own a pool. Counting per pool undercounts every other path, so a+# mixed deployment would silently never hit the cap.+_inflight: dict[str, int] = {}+_inflight_total = 0+_inflight_lock = threading.Lock()+++def _inflight_key(config: dict) -> str:+ try:+ account, region = _account_region_from_function_config(config)+ except Exception:+ account, region = get_account_id(), get_region()+ return f"{account}:{region}:{config.get('FunctionName', '?')}:{config.get('Version', '$LATEST')}"+++def _acquire_execution_slot(func: dict, config: dict):+ """Reserve one unit of concurrency, or None when a cap is reached.++ Never blocks. Waiting for a unit held by your own caller is precisely the+ deadlock this module exists to avoid: a handler that invokes itself would+ wait forever on its own slot.++ ``ReservedConcurrentExecutions`` is stored as a bare int on the *function+ record* under "concurrency" (see _get_function_response), not on the config,+ so both are consulted rather than assuming either shape.+ """+ global _inflight_total+ reserved = (func or {}).get("concurrency")+ if isinstance(reserved, dict):+ reserved = reserved.get("ReservedConcurrentExecutions")+ if reserved is None:+ reserved = config.get("ReservedConcurrentExecutions")+ key = _inflight_key(config)+ with _inflight_lock:+ if reserved and _inflight.get(key, 0) >= int(reserved):+ return None, "function"+ if _ACCOUNT_CONCURRENCY_CAP > 0 and _inflight_total >= _ACCOUNT_CONCURRENCY_CAP:+ return None, "account"+ _inflight[key] = _inflight.get(key, 0) + 1+ _inflight_total += 1+ return key, None+++def _release_execution_slot(key: str) -> None:+ global _inflight_total+ if key is None:+ return+ with _inflight_lock:+ remaining = _inflight.get(key, 0) - 1+ if remaining > 0:+ _inflight[key] = remaining+ else:+ _inflight.pop(key, None)+ _inflight_total = max(0, _inflight_total - 1)+++def executions_in_flight() -> tuple[int, dict]:+ """(total, per-function) invocations currently executing, any executor."""+ with _inflight_lock:+ return _inflight_total, dict(_inflight)++def _execute_function(func: dict, event: dict) -> dict:"""Dispatch an invocation to the right executor and emit CloudWatch Logs.@@ -4054,11 +4207,45 @@ def _execute_function(func: dict, event: dict) -> dict:"""config = func.get("config") or funcif _function_config_scope_mismatch(config):+ # Re-entered below under the right scope; counting here would double-book+ # the slot for a single invocation.return _run_with_function_config_scope(config, _execute_function, func, event)request_id = new_uuid()started = time.time()+ # Concurrency is accounted here, the one point every executor passes+ # through, so warm / docker / provided / proxy / one-off all draw on the+ # same denominator. Accounting inside an executor's own worker pool would+ # only ever see that executor's share.+ slot, limit = _acquire_execution_slot(func, config)+ if slot is None:+ fn_name = config.get("FunctionName", "unknown")+ _emit_lambda_metrics(fn_name, duration_ms=0.0, error=False, throttle=True)+ # AWS models exactly one throttle shape for Invoke —+ # TooManyRequestsException with a Reason distinguishing the function+ # limit from the account limit. Anything else is a code string AWS does+ # not use, which boto3 surfaces as a bare ClientError that+ # `except lambda.exceptions.TooManyRequestsException` will not catch.+ if limit == "function":+ return _throttle_response(+ reason_code="ReservedFunctionConcurrentInvocationLimitExceeded",+ msg=f"Rate Exceeded: function {fn_name} at ReservedConcurrentExecutions",+ )+ return _throttle_response(+ reason_code="ConcurrentInvocationLimitExceeded",+ msg=f"Rate Exceeded: account concurrency cap {_ACCOUNT_CONCURRENCY_CAP} reached",+ )+ try:+ return _execute_function_dispatch(func, config, event, request_id, started)+ finally:+ _release_execution_slot(slot)+++def _execute_function_dispatch(func: dict, config: dict, event: dict,+ request_id: str, started: float) -> dict:+ """Executor selection for a single accounted invocation."""+# Proxy mode wins over every other executor: the function is bound to a# user-managed container, ministack only forwards the event.proxy_url = _proxy_url_for(config)@@ -4230,8 +4417,21 @@ def _execute_function_warm(func: dict, event: dict) -> dict:func_name = config.get("FunctionName", "unknown")qualifier = config.get("Version", "$LATEST")+ # Concurrent invocations take separate execution environments, as on AWS.+ # ReservedConcurrentExecutions caps the function; the account cap is checked+ # across every function. Both report a throttle rather than queueing —+ # blocking for a free worker would deadlock a handler that invokes itself.+ # Concurrency (ReservedConcurrentExecutions / account cap) is already+ # accounted upstream in _execute_function, for every executor. What is left+ # here is purely the warm-subprocess resource ceiling.+ _ensure_reaper_thread()+ worker, reason = acquire_worker(func_name, config, code_zip, qualifier=qualifier)+ if worker is None and reason == "func_cap":+ return _throttle_response(+ reason_code="ReservedFunctionConcurrentInvocationLimitExceeded",+ msg=f"Rate Exceeded: function {func_name} warm-worker ceiling reached",+ )try:- worker = get_or_create_worker(func_name, config, code_zip, qualifier=qualifier)# Inject X-Ray trace header into the event so the worker bootstrap# can set ``_X_AMZN_TRACE_ID`` in os.environ before calling the# handler. Per-invocation, not bake-time, so it can't live in the@@ -4265,11 +4465,14 @@ def _execute_function_warm(func: dict, event: dict) -> dict:except Exception as e:logger.error("Warm worker execution error for %s: %s", func_name, e)invalidate_worker(func_name, qualifier=qualifier, account=get_account_id(), region=get_region())+ worker = None # invalidate_worker already dropped it from the poolreturn {"body": {"errorMessage": str(e), "errorType": type(e).__name__},"error": True,"log": "",}+ finally:+ release_worker(worker)def _execute_function_provided(func: dict, event: dict) -> dict:@@ -6853,7 +7056,8 @@ async def handle_function_url_request(event = _build_function_url_event(url_id, account_id, region, method, path, headers, body, query_params)exec_record = _execution_record_for_config(func_data, func_config)- result = await asyncio.to_thread(_execute_function_with_config_scope, exec_record, event)+ result = await run_reentrant(_execute_function_with_config_scope, exec_record, event,+ thread_name="ministack-lambda-url")if cfg.get("InvokeMode") == "RESPONSE_STREAM" and not result.get("error") and not result.get("throttle"):return _function_url_stream_response(result, cors_headers)diff --git a/ministack/services/mwaa.py b/ministack/services/mwaa.pyindex b618ff6f..8f432f0f 100644--- a/ministack/services/mwaa.py+++ b/ministack/services/mwaa.py@@ -21,7 +21,9 @@ import socketimport threadingimport time+from ministack.core import container_reaperfrom ministack.core.arn import ArnParseError, parse_arn+from ministack.core.concurrency import run_offloopfrom ministack.core.persistence import load_statefrom ministack.core.responses import (AccountRegionScopedDict,@@ -134,12 +136,17 @@ def _validate_source_bucket_arn(bucket_arn: str):return None+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a+# slow or wedged daemon into a minutes-long stall on a request path.+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))++def _get_docker():global _dockerif _docker is None:try:import docker- _docker = docker.from_env()+ _docker = docker.from_env(timeout=_DOCKER_TIMEOUT)except Exception:passreturn _docker@@ -347,7 +354,7 @@ def _start_airflow_container_in_scope(account_id, region, env_name, env):for stale_name in (container_name, legacy_container_name):try:existing = docker_client.containers.get(stale_name)- existing.remove(force=True)+ existing.remove(force=True, v=True)except Exception:pass@@ -358,7 +365,7 @@ def _start_airflow_container_in_scope(account_id, region, env_name, env):environment=container_env,ports={f"{container_port}/tcp": host_port},name=container_name,- labels={"ministack": "mwaa", "region": region, "env_name": env_name},+ labels=container_reaper.own_labels("mwaa", region=region, env_name=env_name),)if ms_network:@@ -525,7 +532,7 @@ def _delete_environment(method, path, headers, body, query_params):try:container = docker_client.containers.get(container_id)container.stop(timeout=5)- container.remove(force=True)+ container.remove(force=True, v=True)except Exception:logger.debug("MWAA: container cleanup for %s failed (may be already gone)", name)@@ -714,13 +721,15 @@ async def handle_request(method, path, headers, body, query_params):# CRUD on /environments/{Name}if clean_path.startswith("/environments/"):if method == "PUT":- return _create_environment(method, path, headers, body, query_params)+ # Spawns the Airflow container: a heavyweight image, slow start.+ # Off-loop; Airflow does not call back into MiniStack.+ return await run_offloop(_create_environment, method, path, headers, body, query_params)if method == "GET":return _get_environment(method, path, headers, body, query_params)if method == "PATCH":- return _update_environment(method, path, headers, body, query_params)+ return await run_offloop(_update_environment, method, path, headers, body, query_params)if method == "DELETE":- return _delete_environment(method, path, headers, body, query_params)+ return await run_offloop(_delete_environment, method, path, headers, body, query_params)return error_response_json("ValidationException",f"Unknown MWAA path: {method} {path}", 400)@@ -735,8 +744,20 @@ def reset():try:container = docker_client.containers.get(container_id)container.stop(timeout=5)- container.remove(force=True)+ container.remove(force=True, v=True)except Exception:pass_release_port(env.get("_host_port"))_environments.clear()+++def _live_container_ids():+ ids = set()+ for _key, env in _environments.all_items():+ cid = env.get("_docker_container_id")+ if cid:+ ids.add(cid)+ return ids+++container_reaper.register_live_ids("mwaa", _live_container_ids)diff --git a/ministack/services/opensearch.py b/ministack/services/opensearch.pyindex 9e85706e..af90e718 100644--- a/ministack/services/opensearch.py+++ b/ministack/services/opensearch.py@@ -18,6 +18,7 @@ State is account- and region-scoped via AccountRegionScopedDict, except tagswhich stay account-scoped because their ARN keys embed region."""+import contextvarsimport copyimport jsonimport logging@@ -26,7 +27,9 @@ import reimport threadingimport time+from ministack.core import container_reaperfrom ministack.core.arn import ArnParseError, parse_arn+from ministack.core.concurrency import resource_lock, run_offloop, spawn_backgroundfrom ministack.core.persistence import load_statefrom ministack.core.responses import (AccountRegionScopedDict,@@ -141,7 +144,7 @@ def reset():):try:c.stop(timeout=2)- c.remove(force=True)+ c.remove(force=True, v=True)except Exception:pass@@ -307,12 +310,17 @@ def _now() -> int:# Docker container management# ---------------------------------------------------------------------------+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a+# slow or wedged daemon into a minutes-long stall on a request path.+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))++def _get_docker():global _docker_clientif _docker_client is None:try:import docker- _docker_client = docker.from_env()+ _docker_client = docker.from_env(timeout=_DOCKER_TIMEOUT)except Exception:passreturn _docker_client@@ -326,6 +334,33 @@ def _dashboards_container_name(domain_name: str) -> str:return f"ministack-opensearch-dashboards-{get_region()}-{domain_name}"+def _image_is_local(docker_client, image: str) -> bool:+ """True when the image is already pulled.++ ``containers.run`` auto-pulls on ImageNotFound, so a cold cache would block+ CreateDomain for the length of the pull — the OpenSearch image is ~1 GB.+ AWS reports ``Processing`` and provisions asynchronously.+ """+ try:+ docker_client.images.get(image)+ return True+ except Exception:+ return False+++def _remove_container_by_id(cid: str) -> None:+ """Best-effort teardown of a container we started but no longer own."""+ docker = _get_docker()+ if not docker or not cid:+ return+ try:+ c = docker.containers.get(cid)+ c.stop(timeout=2)+ c.remove(force=True, v=True)+ except Exception:+ pass++def _spawn_dataplane(domain_name: str, engine_version: str):"""Spawn an OpenSearch container.@@ -358,6 +393,9 @@ def _spawn_dataplane(domain_name: str, engine_version: str):image = apply_image_prefix(DEFAULT_IMAGE)labels = {+ # Standard label so the boot/shutdown orphan reap finds these; the+ # com.ministack.* keys below are what the service's own lookups use.+ **container_reaper.own_labels("opensearch"),"com.ministack.service": "opensearch","com.ministack.domain": domain_name,"com.ministack.region": get_region(),@@ -441,7 +479,7 @@ def _teardown_dataplane(rec: dict) -> None:try:c = docker.containers.get(cid)c.stop(timeout=2)- c.remove(force=True)+ c.remove(force=True, v=True)except Exception:pass@@ -455,7 +493,7 @@ def _teardown_named_dataplane(domain_name: str) -> None:try:c = docker.containers.get(name)c.stop(timeout=2)- c.remove(force=True)+ c.remove(force=True, v=True)except Exception:pass@@ -568,7 +606,15 @@ def _new_domain_record(name, payload):ebs_opts = _default_ebs_options(payload.get("EBSOptions"))access_policies = payload.get("AccessPolicies", "")- host, port, cid, dash_endpoint, dash_cid = _spawn_dataplane(name, engine_version)+ _dc = _get_docker() if DATAPLANE_ENABLED else None+ _deferred = bool(_dc) and not _image_is_local(_dc, apply_image_prefix(DEFAULT_IMAGE))+ if _deferred:+ # Cold image cache: the OpenSearch image is ~1 GB and pulling it inside+ # CreateDomain would block the API for minutes. AWS reports+ # Processing=True and provisions asynchronously; do the same.+ host, port, cid, dash_endpoint, dash_cid = "localhost", BASE_PORT, None, None, None+ else:+ host, port, cid, dash_endpoint, dash_cid = _spawn_dataplane(name, engine_version)endpoint = f"{host}:{port}"rec = {@@ -578,7 +624,7 @@ def _new_domain_record(name, payload):"Created": True,"Deleted": False,"Endpoint": endpoint,- "Processing": False,+ "Processing": _deferred,"UpgradeProcessing": False,"EngineVersion": engine_version,"ClusterConfig": cluster_cfg,@@ -630,6 +676,47 @@ def _new_domain_record(name, payload):_set_vpc_options(rec, payload.get("VPCOptions"))if dash_endpoint:rec["DashboardEndpoint"] = dash_endpoint+ if _deferred:+ # Pull + start off the request path. The domain reports Processing=True+ # until this lands, exactly as AWS does.+ created_record = rec++ def _finish_domain_start():+ try:+ h, pt, cid2, _dep, _dcid = _spawn_dataplane(name, engine_version)+ except Exception:+ logger.warning("OpenSearch: deferred dataplane start failed for %s", name)+ h = pt = cid2 = None+ # Publish under the lock: a DeleteDomain must not land between the+ # existence check and the write. Identity, not truthiness — a domain+ # recreated under the same name is a different resource.+ with resource_lock("opensearch", name):+ live = _domains.get(name)+ orphaned = live is not created_record or live.get("_deleting")+ if not orphaned:+ if h:+ # Both keys. Readers prefer `_Endpoint`, and create left+ # it at the `localhost:BASE_PORT` placeholder, so+ # writing only `Endpoint` left Describe reporting an+ # address nothing listens on — the same defect reported+ # against ElastiCache.+ endpoint = f"{h}:{pt}"+ live["Endpoint"] = endpoint+ live["_Endpoint"] = endpoint+ live["_container_id"] = cid2+ live["_ContainerId"] = cid2+ live["Processing"] = False+ if orphaned and cid2:+ # Nothing references it and the reaper never touches a running+ # container. Outside the lock: Docker calls never hold it.+ _remove_container_by_id(cid2)++ # Handed to the caller rather than started here: this function builds+ # the record, and `_domains[name]` is not assigned until afterwards. A+ # finisher started now can win that race, find no record, and tear down+ # the container it just created — the domain then never becomes+ # available. Started once the record is registered.+ rec["_deferred_start"] = _finish_domain_startreturn rec@@ -759,6 +846,12 @@ def create_domain_record(payload, compatibility_properties=None):_tags[rec["ARN"]] = tagselse:_tags.pop(rec["ARN"], None)+ # Only now — the finisher looks the record up by name and reclaims what+ # it started if it is not there, so it must not run before this.+ deferred_start = rec.pop("_deferred_start", None)+ if deferred_start is not None:+ spawn_background(contextvars.copy_context().run, deferred_start,+ thread_name=f"ministack-opensearch-start-{name}")return recexcept Exception:if rec is not None:@@ -825,7 +918,12 @@ def update_domain_from_cloudformation(name, payload, compatibility_properties):def delete_domain_record(name, missing_ok=False):"""Delete all domain-owned state and containers, optionally idempotently."""- rec = _domains.pop(name, None)+ # Claim under the lock so a deferred start cannot publish into a record we+ # are removing. Teardown itself stays outside — it calls Docker.+ with resource_lock("opensearch", name):+ rec = _domains.pop(name, None)+ if rec:+ rec["_deleting"] = Trueif not rec and not missing_ok:raise OpenSearchServiceError(404, "ResourceNotFoundException", f"Domain not found: {name}"@@ -1255,7 +1353,7 @@ async def handle_request(method, path, headers, body_bytes, query_params):if method == "GET":return _describe_domain_config(name)if method == "POST":- return _update_domain_config(name, payload)+ return await run_offloop(_update_domain_config, name, payload)m = _DOMAIN_PROGRESS_RE.match(path)if method == "GET" and m:@@ -1263,13 +1361,30 @@ async def handle_request(method, path, headers, body_bytes, query_params):m = _DOMAIN_RE.match(path)if method == "POST" and m and m.group("name") is None:- return _create_domain(payload)+ # Spawns the OpenSearch (and Dashboards) container: seconds warm,+ # minutes on a cold pull, and it holds the loop for every other+ # service meanwhile. Shared pool — an OpenSearch node never calls+ # back into MiniStack, so this cannot re-enter.+ return await run_offloop(_create_domain, payload)if m and m.group("name"):name = m.group("name")if method == "GET":return _describe_domain(name)if method == "DELETE":- return _delete_domain(name)+ return await run_offloop(_delete_domain, name)return _error(400, "InvalidAction",f"OpenSearch operation not implemented: {method} {path}")+++def _live_container_ids():+ ids = set()+ for _key, rec in _domains.all_items():+ for field in ("_container_id", "_dashboards_container_id"):+ cid = rec.get(field)+ if cid:+ ids.add(cid)+ return ids+++container_reaper.register_live_ids("opensearch", _live_container_ids)diff --git a/ministack/services/rds.py b/ministack/services/rds.pyindex b735398e..fab8f348 100644--- a/ministack/services/rds.py+++ b/ministack/services/rds.py@@ -46,7 +46,9 @@ import timefrom urllib.parse import parse_qsfrom xml.sax.saxutils import escape as _esc+from ministack.core import container_reaperfrom ministack.core.arn import ArnParseError, parse_arn+from ministack.core.concurrency import resource_lock, run_offloop, spawn_backgroundfrom ministack.core.persistence import load_statefrom ministack.core.responses import (AccountRegionScopedDict,@@ -67,6 +69,11 @@ from ministack.services.rds_mysql_compat import (logger = logging.getLogger("rds")+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a+# slow or wedged daemon into a minutes-long stall on a request path.+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))++REGION = os.environ.get("MINISTACK_REGION", "us-east-1")_MINISTACK_HOST = os.environ.get("MINISTACK_HOST", "localhost")BASE_PORT = int(os.environ.get("RDS_BASE_PORT", "15432"))@@ -1037,7 +1044,7 @@ def _start_cluster_shared_container(cluster_id, cluster, remove_stale=False):ports={f"{container_port}/tcp": host_port},name=container_name,labels={- "ministack": "rds",+ **container_reaper.own_labels("rds"),"cluster_id": cluster_id,"account_id": get_account_id(),"region": get_region(),@@ -1913,6 +1920,21 @@ def _restart_cluster_shared_container(cluster_id, cluster):}+def _image_is_local(docker_client, image: str) -> bool:+ """True when the image is already pulled.++ ``containers.run`` auto-pulls on ImageNotFound, so calling it with a cold+ cache blocks the request for as long as the pull takes — minutes for the+ database and Airflow images. AWS returns ``creating`` in milliseconds and+ provisions asynchronously, so a missing image must not be pulled inline.+ """+ try:+ docker_client.images.get(image)+ return True+ except Exception:+ return False++def _start_rds_container_for_instance(db_id, instance):"""Re-spin (or re-attach to) the Docker container for a restored instance.@@ -2001,7 +2023,7 @@ def _start_rds_container_for_instance(db_id, instance):ports={f"{container_port}/tcp": host_port},name=container_name,labels={- "ministack": "rds",+ **container_reaper.own_labels("rds"),"db_id": db_id,"account_id": get_account_id(),"region": get_region(),@@ -2027,7 +2049,27 @@ def _start_rds_container_for_instance(db_id, instance):instance["DBInstanceStatus"] = "failed"return- instance["_docker_container_id"] = container.id+ # Check-and-publish under the lock, so a DeleteDBInstance cannot land in the+ # gap between them and leave this container referenced by nothing. The check+ # has to be unconditional — the MySQL readiness path below has its own, but+ # Postgres would otherwise leave the container running forever.+ with resource_lock("rds", db_id):+ current = _instances.get(db_id)+ orphaned = current is not instance or instance.get("_deleting")+ if not orphaned:+ instance["_docker_container_id"] = container.id+ if orphaned:+ # Deleted (or replaced) while we were pulling and starting. Reclaim the+ # container: no record references it, and the periodic reaper never+ # touches a *running* container, so nothing else ever would. Outside the+ # lock — Docker calls never hold it.+ logger.info("RDS: instance %s vanished during start; reclaiming container", db_id)+ try:+ container.stop(timeout=2)+ container.remove(force=True, v=True)+ except Exception:+ pass+ returninternal_host = Noneinternal_port = None@@ -2062,6 +2104,15 @@ def _start_rds_container_for_instance(db_id, instance):_instances.get(db_id) is not instanceor instance.get("_docker_container_id") != container.id):+ # The instance was deleted (or replaced) while we were pulling and+ # starting. Reclaim what we started: nothing else can, because the+ # periodic reaper never touches a *running* container and no record+ # references this id any more.+ try:+ container.stop(timeout=2)+ container.remove(force=True, v=True)+ except Exception:+ passreturninstance["DBInstanceStatus"] = "available"logger.info("RDS: respawned container %s for instance %s",@@ -2073,7 +2124,7 @@ def _get_docker():if _docker is None:try:import docker- _docker = docker.from_env()+ _docker = docker.from_env(timeout=_DOCKER_TIMEOUT)except Exception:passreturn _docker@@ -2964,7 +3015,7 @@ def _flatten_json_request_params(params, data):params[f"Filters.member.{i}.Values.member.{j}"] = [str(v)]-async def handle_request(method, path, headers, body, query_params):+def _handle_request_sync(method, path, headers, body, query_params):params = dict(query_params)if method == "POST" and body:raw = body if isinstance(body, str) else body.decode("utf-8-sig", errors="replace")@@ -3454,6 +3505,7 @@ def _create_db_instance_impl(p):readiness_master_pass = master_passms_network = None+ deferred_container_start = Falsedocker_client = _get_docker()reader_launch = Noneif (@@ -3553,7 +3605,16 @@ def _create_db_instance_impl(p):image, env, container_port, data_path = _docker_image_for_engine(engine, engine_version, master_user, master_pass, db_name)- if image:+ if image and not _image_is_local(docker_client, image):+ # Cold image cache: pulling here would block CreateDBInstance for+ # minutes (measured: 187s for postgres:15-alpine). AWS returns+ # `creating` at once, so allocate the port, defer the whole start to+ # the background finaliser below, and let DescribeDBInstances show+ # `available` when it lands.+ host_port = _next_port()+ endpoint_port = host_port+ deferred_container_start = True+ elif image:try:# Create path: the `instance` dict doesn't exist yet# (it's built ~70 lines below). Allocate a fresh free port@@ -3567,7 +3628,7 @@ def _create_db_instance_impl(p):ports={f"{container_port}/tcp": host_port},name=_rds_docker_name(db_id),labels={- "ministack": "rds",+ **container_reaper.own_labels("rds"),"db_id": db_id,"account_id": get_account_id(),"region": get_region(),@@ -3629,6 +3690,10 @@ def _create_db_instance_impl(p):"DBSubnetGroupArn": f"arn:aws:rds:{get_region()}:{get_account_id()}:subgrp:{subnet_group_name}",})instance_status = "creating" if real_container_started else "available"+ if deferred_container_start:+ # The container is being pulled and started in the background; AWS+ # reports `creating` until provisioning completes.+ instance_status = "creating"if parent and not parent.get("_shared_container_ready", True):instance_status = "creating"if parent and start_result.get("failed"):@@ -3737,6 +3802,18 @@ def _create_db_instance_impl(p):_instances[db_id] = instance_register_instance_in_cluster(instance)+ if deferred_container_start:+ # Cold image cache. Pull and start on a background thread — the same+ # finaliser persistence restore uses — so CreateDBInstance returns now+ # with status="creating", as AWS does, instead of blocking on the pull.+ ctx = contextvars.copy_context()++ def _deferred_start():+ _start_rds_container_for_instance(db_id, instance)++ spawn_background(ctx.run, _deferred_start,+ thread_name=f"ministack-rds-start-{db_id}")+if real_container_started and reader_launch:# Replicating readers have their own readiness contract (wait for# the writer, provision replication, wait for the standby), so they@@ -4002,15 +4079,35 @@ def _delete_db_instance(p):else:_stop_cluster_shared_container(shared_cluster_id, cluster)+ # Tombstone first, under the lock: a deferred start still pulling would+ # otherwise publish its container id after we read it here, and the record+ # is gone by the time it lands — leaving a running container nothing+ # references. With the tombstone that finisher reclaims what it started.+ with resource_lock("rds", instance_id):+ instance["_deleting"] = True+ owned_id = instance.get("_docker_container_id") if _instance_owns_container(instance) else None+docker_client = _get_docker()- if docker_client and _instance_owns_container(instance):- try:- c = docker_client.containers.get(instance["_docker_container_id"])- c.stop(timeout=5)- c.remove(v=True)- logger.info("RDS: removed container for %s", instance_id)- except Exception as e:- logger.warning("RDS: failed to remove container for %s: %s", instance_id, e)+ if docker_client:+ # By id, else by the deterministic name — the id may never have been+ # published if the delete beat the deferred start.+ for locate in (+ (lambda: docker_client.containers.get(owned_id)) if owned_id else None,+ lambda: docker_client.containers.get(_rds_docker_name(instance_id)),+ ):+ if locate is None:+ continue+ try:+ c = locate()+ except Exception:+ continue+ try:+ c.stop(timeout=5)+ c.remove(v=True)+ logger.info("RDS: removed container for %s", instance_id)+ except Exception as e:+ logger.warning("RDS: failed to remove container for %s: %s", instance_id, e)+ breakskip_snapshot = _p(p, "SkipFinalSnapshot") == "true"final_snap_id = _p(p, "FinalDBSnapshotIdentifier")@@ -7644,3 +7741,39 @@ except Exception:logging.getLogger(__name__).exception("Failed to restore persisted state; continuing with fresh store")+++async def handle_request(method, path, headers, body, query_params):+ """Dispatch off the event loop.++ Request paths here reach the Docker daemon (container create/start/stop/+ inspect), which blocks for as long as the daemon takes. Measured on ECS: a+ cached-image container start held the loop for 7.3s, during which the health+ endpoint — the cheapest request in the process — could not be served.++ Uses the shared pool: the containers started here (database / cache / search+ engines) never call back into MiniStack, so this cannot re-enter.+ """+ return await run_offloop(+ _handle_request_sync, method, path, headers, body, query_params)+++def _live_container_ids():+ """Container ids still owned by a live DB instance or cluster.++ A stopped instance still owns its (exited) container — StartDBInstance must+ be able to restart it — so it is reported here and never reaped.+ """+ ids = set()+ for _key, inst in _instances.all_items():+ cid = inst.get("_docker_container_id")+ if cid:+ ids.add(cid)+ for _key, cl in _clusters.all_items():+ cid = cl.get("_shared_container_id")+ if cid:+ ids.add(cid)+ return ids+++container_reaper.register_live_ids("rds", _live_container_ids)diff --git a/ministack/services/stepfunctions.py b/ministack/services/stepfunctions.pyindex b5b14f7e..9600519e 100644--- a/ministack/services/stepfunctions.py+++ b/ministack/services/stepfunctions.py@@ -39,7 +39,8 @@ from concurrent.futures import wait as futures_waitfrom datetime import datetime, timezonefrom ministack.core.arn import ArnParseError, parse_arn-from ministack.core.persistence import PERSIST_STATE, load_state+from ministack.core.concurrency import run_reentrant+from ministack.core.persistence import load_statefrom ministack.core.responses import (AccountRegionScopedDict,AccountScopedDict,@@ -236,6 +237,12 @@ def _finalize_response(response):# Entry point# ---------------------------------------------------------------------------+# Actions that run a state machine to completion inside the request. They+# re-enter MiniStack (Lambda invokes, aws-sdk integrations), so they run on a+# dedicated thread rather than holding the event loop those nested calls need.+_OFF_LOOP_ACTIONS = {"StartSyncExecution", "TestState"}++async def handle_request(method, path, headers, body, query_params):target = headers.get("x-amz-target", "")action = target.split(".")[-1] if "." in target else ""@@ -286,6 +293,15 @@ async def handle_request(method, path, headers, body, query_params):return error_response_json("InvalidAction", f"Unknown action: {action}", 400)if action == "GetActivityTask":return _finalize_response(await _get_activity_task(data))+ if action in _OFF_LOOP_ACTIONS:+ # These run a state machine to completion inline. Every task state calls+ # back into MiniStack — Lambda invokes, aws-sdk service integrations —+ # so this must not hold the event loop: the nested calls need it, and a+ # service handler that awaits (Docker work runs off-loop now) cannot be+ # driven from the loop thread at all. Dedicated thread, not the shared+ # pool, because the work re-enters the server.+ return _finalize_response(+ await run_reentrant(handler, data, thread_name="ministack-sfn-sync-exec"))return _finalize_response(handler(data))@@ -4386,25 +4402,14 @@ def _dispatch_aws_sdk_json(service_info, service_name, action, input_data):),}- # Service handlers are async def but perform no real I/O, so we can- # drive the coroutine synchronously — this avoids conflicts with the- # already-running asyncio event loop.- coro = handler("POST", "/", headers, body, {})- try:- coro.send(None)- except StopIteration as stop:- status, resp_headers, resp_body = stop.value- else:- # If the coroutine didn't finish in one step it truly needs async;- # fall back to the event loop (only reachable if a handler awaits).- coro.close()- loop = asyncio.new_event_loop()- try:- status, resp_headers, resp_body = loop.run_until_complete(- handler("POST", "/", headers, body, {})- )- finally:- loop.close()+ # Service handlers are async def and may now genuinely await (services that+ # drive Docker run that work off the event loop), so they cannot be driven+ # by hand with ``coro.send(None)``: the first step would execute with no+ # running loop and raise "no running event loop". Schedule on the loop that+ # is actually serving and wait here. This runs on a Step Functions execution+ # thread, never on the loop itself, so the wait cannot deadlock.+ status, resp_headers, resp_body = app.call_service_handler_sync(+ handler, "POST", "/", headers, body, {})decoded = resp_body.decode("utf-8") if isinstance(resp_body, bytes) else resp_bodyresult = json.loads(decoded) if decoded else {}@@ -4989,20 +4994,9 @@ def _dispatch_aws_sdk_query(service_info, service_name, action, input_data):),}- coro = handler("POST", "/", headers, body, {})- try:- coro.send(None)- except StopIteration as stop:- status, resp_headers, resp_body = stop.value- else:- coro.close()- loop = asyncio.new_event_loop()- try:- status, resp_headers, resp_body = loop.run_until_complete(- handler("POST", "/", headers, body, {})- )- finally:- loop.close()+ # Scheduled on the serving loop — see _dispatch_aws_sdk_json.+ status, resp_headers, resp_body = app.call_service_handler_sync(+ handler, "POST", "/", headers, body, {})decoded = resp_body.decode("utf-8") if isinstance(resp_body, bytes) else resp_body@@ -5092,20 +5086,9 @@ def _dispatch_aws_sdk_rest_json(service_info, service_name, action, input_data):),}- coro = handler("POST", path, headers, body, {})- try:- coro.send(None)- except StopIteration as stop:- status, resp_headers, resp_body = stop.value- else:- coro.close()- loop = asyncio.new_event_loop()- try:- status, resp_headers, resp_body = loop.run_until_complete(- handler("POST", path, headers, body, {})- )- finally:- loop.close()+ # Scheduled on the serving loop — see _dispatch_aws_sdk_json.+ status, resp_headers, resp_body = app.call_service_handler_sync(+ handler, "POST", path, headers, body, {})decoded = resp_body.decode("utf-8") if isinstance(resp_body, bytes) else resp_body@@ -5226,26 +5209,11 @@ def _dispatch_aws_sdk_lambda_rest(service_info, service_name, action, input_data),}- # Drive the async handler synchronously. SFN state execution runs inside- # the request's event loop, so spawning a fresh loop here would raise- # "Cannot run the event loop while another loop is running". The Lambda- # REST handlers we dispatch to here only touch in-memory dicts and never- # await, so a single ``coro.send(None)`` completes via ``StopIteration``- # with the response tuple.- coro = handler(method, path, headers, body, query_params)- try:- coro.send(None)- except StopIteration as stop:- status, resp_headers, resp_body = stop.value- else:- # Fallback for an async handler that does await — extremely rare on- # this dispatch surface, but guard so we don't return None silently.- coro.close()- raise _ExecutionError(- "States.Runtime",- f"aws-sdk:{service_name}:{action} handler awaited unexpectedly; "- "cannot drive from sync SFN executor",- )+ # Scheduled on the serving loop — see _dispatch_aws_sdk_json. This replaces+ # a guard that failed the execution outright when a handler awaited; that is+ # now an ordinary, supported case.+ status, resp_headers, resp_body = app.call_service_handler_sync(+ handler, method, path, headers, body, query_params)decoded = resp_body.decode("utf-8") if isinstance(resp_body, bytes) else resp_bodyif status >= 400:@@ -5577,20 +5545,9 @@ def _dispatch_aws_sdk_rest_xml(service_info, service_name, action, input_data):# passing the raw path with "?..." would treat "?list-type=2" as part of# the bucket name. Send the query-less path; the dict is the source of# truth for query routing.- coro = handler(method, path, headers, body, query_dict)- try:- coro.send(None)- except StopIteration as stop:- status, resp_headers, resp_body = stop.value- else:- coro.close()- loop = asyncio.new_event_loop()- try:- status, resp_headers, resp_body = loop.run_until_complete(- handler(method, path, headers, body, query_dict)- )- finally:- loop.close()+ # Scheduled on the serving loop — see _dispatch_aws_sdk_json.+ status, resp_headers, resp_body = app.call_service_handler_sync(+ handler, method, path, headers, body, query_dict)decoded = resp_body.decode("utf-8") if isinstance(resp_body, bytes) else (resp_body or "")norm_resp_headers = {k.lower(): v for k, v in (resp_headers or {}).items()}diff --git a/ministack/services/transfer.py b/ministack/services/transfer.pyindex 85d16442..4275f195 100644--- a/ministack/services/transfer.py+++ b/ministack/services/transfer.py@@ -38,9 +38,8 @@ import ioimport jsonimport loggingimport os-import reimport time-from typing import AsyncIterator, Optional+from typing import Optionalfrom ministack.core.persistence import load_statefrom ministack.core.responses import (@@ -213,6 +212,16 @@ async def _create_server(data):_servers[sid] = server+ # Bind the shared listener here rather than at boot. Nobody can connect to+ # a server they have not created, so binding on CreateServer is equivalent+ # — and it keeps `asyncssh` (which drags in cryptography + OpenSSL, ~26 MB+ # of heap) out of every MiniStack that never touches Transfer Family.+ # Idempotent, so repeated CreateServer calls cost one dict lookup.+ try:+ await sftp_start()+ except Exception as e:+ logger.warning("SFTP listener failed to start for %s: %s", sid, e)+# If SFTP_PORT_PER_SERVER=1, allocate a dedicated SFTP listener for# this server (matches AWS's per-server endpoint model). Otherwise# the shared :2222 listener handles every server, disambiguating by@@ -1219,6 +1228,19 @@ def _ensure_sftp_runtime() -> None:# ── SFTP lifecycle — called from app.py lifespan + Create/Delete handlers ──+def has_servers() -> bool:+ """True if any Transfer server exists, in any account/region.++ Used at boot to decide whether the SFTP listener has to come up before a+ CreateServer call would bring it up on its own — i.e. only when persistence+ restored servers that a client may connect to immediately.+ """+ try:+ return bool(_servers.to_dict())+ except Exception:+ return False++async def sftp_start() -> None:"""Idempotent: bind the shared SFTP listener on `SFTP_PORT` if enabled."""if not _sftp_enabled():diff --git a/tests/conftest.py b/tests/conftest.pyindex f5d2756b..5675fe02 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -2,10 +2,17 @@Pytest fixtures for MiniStack integration tests."""+import concurrent.futuresimport contextlib+import ioimport osimport socket+import statistics+import threading+import timeimport urllib.request+import uuid+import zipfilefrom urllib.parse import urlparseimport boto3@@ -719,3 +726,171 @@ def dsql():@pytest.fixture(scope="session")def s3tables():return make_client("s3tables")+++class FakeDockerContainer:+ """Container double for tests that observe lifecycle without a daemon."""++ def __init__(self, daemon, name, labels):+ self.daemon, self.name, self.labels = daemon, name, dict(labels or {})+ self.id = f"cid-{name}"+ self.status = "running"+ self.attrs = {"Config": {"Labels": self.labels}, "State": {}, "Created": ""}++ def stop(self, timeout=None):+ self.status = "exited"++ def remove(self, force=False, v=False):+ self.daemon.removed.append(self.name)+ self.daemon.containers._by_name.pop(self.name, None)++ def reload(self):+ pass+++class _FakeContainers:+ def __init__(self, daemon):+ self.daemon = daemon+ self._by_name = {}++ def run(self, image=None, command=None, name=None, labels=None, **kw):+ self._by_name[name] = FakeDockerContainer(self.daemon, name, labels)+ return self._by_name[name]++ def get(self, key):+ for c in self._by_name.values():+ if key in (c.name, c.id):+ return c+ raise RuntimeError(f"no such container: {key}")++ def list(self, all=False, filters=None):+ want = (filters or {}).get("label") or []+ want = [want] if isinstance(want, str) else want+ out = []+ for c in self._by_name.values():+ ok = True+ for sel in want:+ if "=" in sel:+ k, v = sel.split("=", 1)+ ok = ok and c.labels.get(k) == v+ else:+ ok = ok and sel in c.labels+ if ok:+ out.append(c)+ return out+++class FakeDockerDaemon:+ """Enough of the Docker API to assert which containers survive.++ ``images.get`` always raises, so services that check whether an image is+ cached take their deferred/background provisioning path.+ """++ def __init__(self):+ self.containers = _FakeContainers(self)+ self.removed = []+ self.images = self++ def get(self, image):+ raise RuntimeError("image not present")++ def live(self):+ return sorted(self.containers._by_name)+++def fake_docker():+ return FakeDockerDaemon()+++# ---------------------------------------------------------------------------+# Concurrency probes — shared by the per-service re-entrancy tests+# ---------------------------------------------------------------------------++# Enough callers to exhaust a misclassified bounded pool without being slow.+CONCURRENCY_N = 12+# The nested-invoke probe has to exceed the shared worker pool (default 64) or+# starvation never shows: below the pool size every caller gets a slot and a+# broken build passes. Verified — at N=12 the probe passes against 1.4.17,+# which has the bug.+CONCURRENCY_N_NESTED = 70+# Judged on the *median*, never the max. A single slow sample says nothing about+# the event loop: the probe competes with the burst for client connections and+# CI runner CPU, so the worst sample is dominated by queueing on our side.+# Measured spread — a build with the bug blocks the loop for the whole burst and+# every sample is slow (1.4.17: median 6905ms), while a healthy one absorbs it+# (median 118ms, with an occasional multi-second outlier under a saturated+# connection pool). The median separates those by ~60x; the max does not+# separate them at all.+MEDIAN_LOOP_STALL_MS = 2000+++class LoopProbe:+ """Polls /_ministack/health through a burst to detect a blocked event loop.++ Used by the re-entrancy tests: work that can call back into MiniStack must+ run on a dedicated thread, and work that blocks must never run on the loop.+ Both failures show up here as the health endpoint going slow or silent.+ """++ MIN_SAMPLES = 3 # below this an outlier decides the median; too short to judge++ def __init__(self):+ self.samples = []+ self._stop = threading.Event()+ self._thread = None++ def __enter__(self):+ def poll():+ while not self._stop.is_set():+ t0 = time.perf_counter()+ try:+ urllib.request.urlopen(f"{ENDPOINT}/_ministack/health", timeout=15).read()+ self.samples.append((time.perf_counter() - t0) * 1000)+ except Exception:+ self.samples.append(None)+ time.sleep(0.05)++ self._thread = threading.Thread(target=poll, daemon=True)+ self._thread.start()+ return self++ def __exit__(self, *exc):+ self._stop.set()+ if self._thread:+ self._thread.join(timeout=5)++ def assert_responsive(self, what: str):+ served = [s for s in self.samples if s is not None]+ assert served, f"{what}: health never responded — the event loop was blocked"+ if len(served) < self.MIN_SAMPLES:+ return+ median = statistics.median(served)+ assert median < MEDIAN_LOOP_STALL_MS, (+ f"{what}: event loop median {median:.0f}ms "+ f"(worst {max(served):.0f}ms, n={len(self.samples)}, "+ f"unanswered={len(self.samples) - len(served)}) — "+ f"a blocking call is running on the loop"+ )+++def concurrent_burst(fn, items=None, n=CONCURRENCY_N):+ """Run `fn` concurrently over `items` (or over range(n)); return all results."""+ work = list(items) if items is not None else list(range(n))+ with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, len(work))) as ex:+ return list(ex.map(fn, work))+++def make_probe_lambda(lam, src: str, timeout: int = 30) -> str:+ """Deploy a throwaway Python function and return its name."""+ buf = io.BytesIO()+ with zipfile.ZipFile(buf, "w") as z:+ z.writestr("index.py", src)+ name = f"conc-{uuid.uuid4().hex[:10]}"+ lam.create_function(+ FunctionName=name, Runtime="python3.11",+ Role="arn:aws:iam::000000000000:role/lambda-role", Handler="index.handler",+ Code={"ZipFile": buf.getvalue()}, Timeout=timeout,+ )+ return namediff --git a/tests/test_apigatewayv2.py b/tests/test_apigatewayv2.pyindex d190df4b..d3d20464 100644--- a/tests/test_apigatewayv2.py+++ b/tests/test_apigatewayv2.py@@ -6,6 +6,8 @@ import jsonimport osimport threadingimport time+import urllib.error+import urllib.requestimport uuid as _uuid_modimport zipfilefrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServer@@ -13,6 +15,13 @@ from urllib.parse import urlparseimport pytestfrom botocore.exceptions import ClientError+from conftest import (+ CONCURRENCY_N,+ ENDPOINT,+ LoopProbe,+ concurrent_burst,+ make_probe_lambda,+)_endpoint = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566")@@ -4150,3 +4159,43 @@ def test_apigwv2_authorizer_cache_is_bounded():finally:cache.clear()cache.update(saved)+++# ---------------------------------------------------------------------------+# Re-entrancy under concurrency — the rest of the suite issues one request at+# a time, so a service whose blocking work is misclassified passes serially+# and only wedges under load. These fire N callers at once and assert both+# that every caller completes and that the event loop keeps serving.+# ---------------------------------------------------------------------------+def test_concurrent_apigw_lambda_proxy_requests(apigw, lam):+ """N simultaneous HTTP-API requests, each executing a Lambda, must all serve."""+ fn = make_probe_lambda(lam, (+ "import time\n"+ "def handler(event, context):\n"+ " time.sleep(0.4)\n"+ " return {'statusCode': 200, 'body': 'ok'}\n"+ ))+ api = apigw.create_api(Name=f"conc-{uuid.uuid4().hex[:8]}", ProtocolType="HTTP",+ Target=f"arn:aws:lambda:us-east-1:000000000000:function:{fn}")+ api_id = api["ApiId"]++ def call(_):+ try:+ r = urllib.request.urlopen(f"{ENDPOINT}/_apigw/{api_id}/", timeout=30)+ return r.status+ except urllib.error.HTTPError as e:+ return e.code+ except Exception:+ return None++ with LoopProbe() as probe:+ codes = concurrent_burst(call)+ probe.assert_responsive("apigw proxy burst")++ # The route wiring differs by MiniStack version; what this test guards is+ # that the server answered every caller rather than wedging.+ assert all(c is not None for c in codes), (+ f"{codes.count(None)}/{CONCURRENCY_N} API Gateway requests got no response at all — "+ f"the Lambda dispatch wedged"+ )diff --git a/tests/test_dsql.py b/tests/test_dsql.pyindex e8b324e0..3cad769b 100644--- a/tests/test_dsql.py+++ b/tests/test_dsql.py@@ -4,17 +4,26 @@ validator unit tests and live data-plane (wire proxy) tests."""import asyncio+import osimport reimport socketimport structimport threadingimport time+from urllib.parse import urlparseimport pytestfrom botocore.exceptions import ClientErrorfrom ministack.core import pgproxy+# Raw HTTP calls here must hit the same server the boto3 fixtures use. Hardcoding+# 4566 silently fails everywhere except a default-port run (CI), which is exactly+# the kind of test that looks green until someone runs the suite on another port.+_ENDPOINT = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566").rstrip("/")+_ENDPOINT_HOST = urlparse(_ENDPOINT).hostname or "localhost"+_ENDPOINT_PORT = urlparse(_ENDPOINT).port or 4566+ID_RE = re.compile(r"^[a-z0-9]{26}$")@@ -244,7 +253,7 @@ class TestClusterLifecycle:for bad in (0, 101):req = urllib.request.Request(- f"http://127.0.0.1:4566/cluster?max-results={bad}",+ f"{_ENDPOINT}/cluster?max-results={bad}",headers={"Host": "dsql.us-east-1.localhost"},)with pytest.raises(urllib.error.HTTPError) as exc:diff --git a/tests/test_ecs.py b/tests/test_ecs.pyindex af2f2008..49f0eb74 100644--- a/tests/test_ecs.py+++ b/tests/test_ecs.py@@ -1,15 +1,14 @@import copy-import ioimport jsonimport osimport timeimport uuid as _uuid_mod-import zipfilefrom types import SimpleNamespacefrom urllib.parse import urlparseimport pytestfrom botocore.exceptions import ClientError+from conftest import LoopProbefrom ministack.services import ecs as ecs_service@@ -454,7 +453,7 @@ def test_ecs_create_service_v2(ecs):ecs.create_cluster(clusterName="ecs-svc-v2c")ecs.register_task_definition(family="ecs-svc-v2td",- containerDefinitions=[{"name": "w", "image": "nginx", "cpu": 64, "memory": 128}],+ containerDefinitions=[{"name": "w", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],)resp = ecs.create_service(cluster="ecs-svc-v2c",@@ -471,7 +470,7 @@ def test_ecs_describe_services_v2(ecs):ecs.create_cluster(clusterName="ecs-ds-v2c")ecs.register_task_definition(family="ecs-ds-v2td",- containerDefinitions=[{"name": "w", "image": "nginx", "cpu": 64, "memory": 128}],+ containerDefinitions=[{"name": "w", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],)ecs.create_service(cluster="ecs-ds-v2c",@@ -495,7 +494,7 @@ def test_ecs_update_service_v2(ecs):ecs.create_cluster(clusterName="ecs-us-v2c")ecs.register_task_definition(family="ecs-us-v2td",- containerDefinitions=[{"name": "w", "image": "nginx", "cpu": 64, "memory": 128}],+ containerDefinitions=[{"name": "w", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],)ecs.create_service(cluster="ecs-us-v2c",@@ -567,7 +566,7 @@ def test_ecs_service_arn_parser_does_not_tail_resolve_invalid_arns(ecs):cluster_arn = ecs.create_cluster(clusterName=cluster)["cluster"]["clusterArn"]ecs.register_task_definition(family="ecs-arn-service-td",- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],)created = ecs.create_service(cluster=cluster,@@ -656,7 +655,7 @@ def test_ecs_service_arn_parser_does_not_tail_resolve_invalid_arns(ecs):def test_ecs_task_definition_arn_parser_does_not_tail_resolve_invalid_arns(ecs):resp = ecs.register_task_definition(family="ecs-arn-td",- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],)task_definition_arn = resp["taskDefinition"]["taskDefinitionArn"]valid = ecs.describe_task_definition(taskDefinition=task_definition_arn)@@ -828,7 +827,7 @@ def test_ecs_timestamps_are_epoch(ecs):# registeredAt might not be present on cluster, test on task defecs.register_task_definition(family="ts-td-v44",- containerDefinitions=[{"name": "app", "image": "nginx", "memory": 256}],+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "memory": 256}],)td = ecs.describe_task_definition(taskDefinition="ts-td-v44")registered_at = td["taskDefinition"].get("registeredAt")@@ -847,7 +846,7 @@ def test_ecs_service_spawns_tasks(ecs):ecs.create_cluster(clusterName=cluster)ecs.register_task_definition(family="svc-spawn-td",- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],)ecs.create_service(cluster=cluster,@@ -872,7 +871,7 @@ def test_ecs_list_services(ecs):ecs.create_cluster(clusterName=cluster)ecs.register_task_definition(family="ls-svc-td",- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],)ecs.create_service(cluster=cluster, serviceName="ls-svc-a", taskDefinition="ls-svc-td", desiredCount=1,@@ -893,7 +892,7 @@ def test_ecs_service_running_count(ecs):ecs.create_cluster(clusterName=cluster)ecs.register_task_definition(family="rc-td",- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],)ecs.create_service(cluster=cluster, serviceName="rc-svc", taskDefinition="rc-td", desiredCount=3,@@ -910,7 +909,7 @@ def test_ecs_service_scale_up(ecs):ecs.create_cluster(clusterName=cluster)ecs.register_task_definition(family="su-td",- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],)ecs.create_service(cluster=cluster, serviceName="su-svc", taskDefinition="su-td", desiredCount=1,@@ -932,7 +931,7 @@ def test_ecs_service_scale_down(ecs):ecs.create_cluster(clusterName=cluster)ecs.register_task_definition(family="sd-td",- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],)ecs.create_service(cluster=cluster, serviceName="sd-svc", taskDefinition="sd-td", desiredCount=3,@@ -997,7 +996,7 @@ def test_ecs_service_delete_stops_tasks(ecs):ecs.create_cluster(clusterName=cluster)ecs.register_task_definition(family="del-td",- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],)ecs.create_service(cluster=cluster, serviceName="del-svc", taskDefinition="del-td", desiredCount=2,@@ -1027,7 +1026,7 @@ def test_ecs_service_scale_to_zero(ecs):ecs.create_cluster(clusterName=cluster)ecs.register_task_definition(family="z-td",- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],)ecs.create_service(cluster=cluster, serviceName="z-svc", taskDefinition="z-td", desiredCount=2,@@ -1050,7 +1049,7 @@ def test_ecs_cluster_task_counts(ecs):ecs.create_cluster(clusterName=cluster)ecs.register_task_definition(family="ct-td",- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],)ecs.create_service(cluster=cluster, serviceName="ct-svc", taskDefinition="ct-td", desiredCount=3,@@ -1364,3 +1363,37 @@ def test_ecs_container_secret_arn_selects_the_requested_region():sm_service.reset()set_request_account_id(original_account)set_request_region(original_region)+++# ---------------------------------------------------------------------------+# Re-entrancy under concurrency — the rest of the suite issues one request at+# a time, so a service whose blocking work is misclassified passes serially+# and only wedges under load. These fire N callers at once and assert both+# that every caller completes and that the event loop keeps serving.+# ---------------------------------------------------------------------------+def test_ecs_run_task_does_not_block_the_loop(ecs):+ """RunTask talks to the Docker daemon; that must never happen on the loop."""+ cluster = f"conc-{_uuid_mod.uuid4().hex[:8]}"+ ecs.create_cluster(clusterName=cluster)+ ecs.register_task_definition(+ family=f"{cluster}-td",+ containerDefinitions=[{"name": "app", "image": "alpine:latest",+ "command": ["sleep", "3600"], "memory": 64, "essential": True}],+ )+ with LoopProbe() as probe:+ try:+ ecs.run_task(cluster=cluster, taskDefinition=f"{cluster}-td", count=1)+ except Exception as exc: # no daemon / image pull refused+ pytest.skip(f"ECS RunTask unavailable in this environment: {exc}")+ finally:+ for arn in ecs.list_tasks(cluster=cluster).get("taskArns", []):+ try:+ ecs.stop_task(cluster=cluster, task=arn)+ except Exception:+ pass+ try:+ ecs.delete_cluster(cluster=cluster)+ except Exception:+ pass+ probe.assert_responsive("ECS RunTask")diff --git a/tests/test_eks.py b/tests/test_eks.pyindex 6a4fb730..812a656a 100644--- a/tests/test_eks.py+++ b/tests/test_eks.py@@ -645,11 +645,14 @@ def test_eks_k3s_run_kwargs_container_name_and_labels_are_region_scoped():assert east["name"] == "ministack-eks-us-east-1-my-cluster"assert west["name"] == "ministack-eks-us-west-2-my-cluster"- assert east["labels"] == {+ # Subset, not equality: containers also carry the ownership labels+ # (`ministack.instance` / `ministack.boot`) that scope reaping to this+ # MiniStack. This test is about region scoping, so it asserts only that.+ assert {"ministack": "eks","cluster_name": "my-cluster","region": "us-east-1",- }+ }.items() <= east["labels"].items()assert west["labels"]["region"] == "us-west-2"diff --git a/tests/test_elasticache.py b/tests/test_elasticache.pyindex bdeb136c..a27cb590 100644--- a/tests/test_elasticache.py+++ b/tests/test_elasticache.py@@ -1732,3 +1732,103 @@ def test_valkey_no_docker_fallback_uses_redis_port(monkeypatch):)assert (host, port) == (elasticache.REDIS_DEFAULT_HOST, elasticache.REDIS_DEFAULT_PORT)assert cid is None+++# ---------------------------------------------------------------------------+# Deferred (cold-image) provisioning — state, not liveness+# ---------------------------------------------------------------------------++def test_elasticache_cold_image_publishes_the_real_endpoint(fake_docker, monkeypatch):+ """Describe must report the port the container actually listens on.++ When the engine image is not cached, CreateCacheCluster returns at once and+ a background finisher starts the container. The finisher wrote+ `ConfigurationEndpoint` while every Describe path reads `_endpoint` and+ `CacheNodes[].Endpoint` — which create had filled with the *engine default*+ port, no container existing yet. So a cold-image create reported+ `localhost:6379` for the rest of its life while the container was on the+ allocated host port, and never self-corrected. Reported by @Areson.+ """+ import threading++ from ministack.core.responses import set_request_account_id, set_request_region+ from ministack.services import elasticache as ec++ set_request_account_id("000000000000")+ set_request_region("us-east-1")+ monkeypatch.setattr(ec, "_get_docker", lambda: fake_docker)+ monkeypatch.setattr(ec, "_image_is_local", lambda *_a: False)+ monkeypatch.setattr(ec, "_spawn_redis_container",+ lambda **kw: ("localhost", 16379, "cid-real"))++ done = threading.Event()+ orig = ec.spawn_background++ def tracked(fn, *a, **kw):+ t = orig(fn, *a, **kw)+ threading.Thread(target=lambda: (t.join(), done.set()), daemon=True).start()+ return t+ monkeypatch.setattr(ec, "spawn_background", tracked)++ ec._create_cache_cluster({"CacheClusterId": ["cold-1"], "Engine": ["redis"],+ "CacheNodeType": ["cache.t3.micro"], "NumCacheNodes": ["1"]})+ assert done.wait(timeout=10), "deferred finisher never completed"++ rec = ec._clusters["cold-1"]+ assert rec["CacheClusterStatus"] == "available"+ assert rec["_endpoint"] == {"Address": "localhost", "Port": 16379}, (+ f"Describe would report {rec['_endpoint']} — nothing is listening there"+ )+ assert rec["CacheNodes"][0]["Endpoint"]["Port"] == 16379+++def test_elasticache_delete_during_deferred_start_leaves_no_container(fake_docker, monkeypatch):+ """A delete while the container is being started must not strand it.++ The finisher checked the record existed and then published the container id+ into it. A delete in that gap removed the record, found no id to tear down,+ and the container ran on referenced by nothing — unreachable by any API+ call. Reported by @Areson.+ """+ import threading++ from ministack.core.responses import set_request_account_id, set_request_region+ from ministack.services import elasticache as ec++ set_request_account_id("000000000000")+ set_request_region("us-east-1")+ monkeypatch.setattr(ec, "_get_docker", lambda: fake_docker)+ monkeypatch.setattr(ec, "_image_is_local", lambda *_a: False)++ spawned, release, finished = threading.Event(), threading.Event(), threading.Event()++ def fake_spawn(**kw):+ name = kw.get("name")+ fake_docker.containers.run(name=name, labels=kw.get("labels"))+ spawned.set()+ release.wait(timeout=10) # hold inside the provisioning window+ return "localhost", 16380, f"cid-{name}"+ monkeypatch.setattr(ec, "_spawn_redis_container", fake_spawn)++ orig = ec.spawn_background++ def tracked(fn, *a, **kw):+ t = orig(fn, *a, **kw)+ threading.Thread(target=lambda: (t.join(), finished.set()), daemon=True).start()+ return t+ monkeypatch.setattr(ec, "spawn_background", tracked)++ ec._create_cache_cluster({"CacheClusterId": ["race-1"], "Engine": ["redis"],+ "CacheNodeType": ["cache.t3.micro"], "NumCacheNodes": ["1"]})+ assert spawned.wait(timeout=10), "container was never started"++ ec._delete_cache_cluster({"CacheClusterId": ["race-1"]})+ release.set()+ assert finished.wait(timeout=10), "finisher never completed"++ assert "race-1" not in ec._clusters+ assert fake_docker.live() == [], (+ f"leaked container(s) {fake_docker.live()} — running, and no API call can reach them"+ )diff --git a/tests/test_lambda.py b/tests/test_lambda.pyindex 6e901f44..74de1769 100644--- a/tests/test_lambda.py+++ b/tests/test_lambda.py@@ -15,6 +15,13 @@ import boto3import pytestfrom botocore.config import Configfrom botocore.exceptions import ClientError+from conftest import (+ CONCURRENCY_N_NESTED,+ ENDPOINT,+ LoopProbe,+ concurrent_burst,+ make_probe_lambda,+)_endpoint = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566").rstrip("/")@@ -1657,7 +1664,6 @@ def test_lambda_esm_sqs_report_batch_item_failures(lam, sqs):def test_lambda_warm_start(lam, apigw):"""Warm worker via API Gateway execute-api: module-level state persists across invocations."""import urllib.request as _urlreq- import uuid as _uuidfname = f"intg-warm-{_uuid_mod.uuid4().hex[:8]}"code = (@@ -2768,7 +2774,6 @@ def test_lambda_invoke_dry_run_returns_204(lam):assert resp["StatusCode"] == 204def test_lambda_layer_publish(lam):- import base64import ioimport zipfile@@ -2827,7 +2832,6 @@ def test_lambda_layer_list_layers(lam):assert "my-test-layer" in namesdef test_lambda_layer_delete_version(lam):- import base64import ioimport zipfile@@ -3052,7 +3056,7 @@ def test_lambda_pool_kill_function_reaps_all_qualifiers():self.removed = Falsedef stop(self, timeout=2):self.stopped = True- def remove(self, force=False):+ def remove(self, force=False, v=False):self.removed = Truestubs = [_StubContainer() for _ in range(3)]@@ -4042,7 +4046,6 @@ def test_lambda_provided_runtime_parallel_invocations():def test_apigwv2_nodejs_lambda_proxy(lam, apigw):"""API Gateway v2 HTTP API should invoke Node.js Lambda via warm worker, not return mock."""import urllib.request as _urlreq- import uuid as _uuidfrom botocore.exceptions import ClientError@@ -7587,7 +7590,6 @@ def test_nodejs_worker_aws_sdk_v3_stub_wire_roundtrip(lam, ssm):router.py undetected by the resolution-only tests)."""import shutil- import uuid as _uuidif not shutil.which("node"):pytest.skip("node not found on PATH")@@ -7696,7 +7698,6 @@ def _create_durable_execution_directly(lam):response header. Returns (function_name, function_arn, dict withDurableExecutionArn + CheckpointToken)."""import base64 as _b64- import json as _jsonfname = f"durable-fn-{_uuid_mod.uuid4().hex[:8]}"try:lam.delete_function(FunctionName=fname)@@ -7738,7 +7739,6 @@ def test_lambda_durable_function_config_round_trip(lam):except Exception:passimport base64 as _b64- import json as _jsonzip_b64 = _b64.b64encode(_make_zip("def handler(e,c): return e")).decode()code, body = _raw_durable("POST", "/2015-03-31/functions", body={"FunctionName": fname,@@ -7870,7 +7870,6 @@ def test_lambda_durable_stop(lam):def test_lambda_durable_list_by_function(lam):fname, fn_arn, rec = _create_durable_execution_directly(lam)try:- from urllib.parse import quotecode, body = _raw_durable("GET", f"/2025-12-01/functions/{fname}/durable-executions")assert code == 200arns = [s["DurableExecutionArn"] for s in body["DurableExecutions"]]@@ -7940,7 +7939,6 @@ def test_lambda_durable_chained_invoke_runs_child(lam):the child function and records the result back into the parent'soperation log."""import base64 as _b64- import json as _jsonparent = f"durable-parent-{_uuid_mod.uuid4().hex[:8]}"child = f"durable-child-{_uuid_mod.uuid4().hex[:8]}"for n in (parent, child):@@ -8881,7 +8879,6 @@ def test_lambda_durable_create_function_durable_config_round_trip_with_update(la"""DurableConfig must survive UpdateFunctionConfiguration that touchesunrelated fields (timeout, memory)."""import base64 as _b64- import json as _jsonfname = f"dur-upd-{_uuid_mod.uuid4().hex[:8]}"try:@@ -10110,7 +10107,7 @@ def test_lambda_keepalive_zero_forces_cold_start(monkeypatch):passdef stop(self, timeout=2):self.stopped = True- def remove(self, force=False):+ def remove(self, force=False, v=False):self.removed = Truestub = _StubContainer()@@ -10139,6 +10136,128 @@ def test_lambda_keepalive_zero_forces_cold_start(monkeypatch):# ---------------------------------------------------------------------------+# Re-entrant invocation — a handler that calls back into the emulator during+# its own invocation, including into its own function.+#+# Every synchronous execution used to hold one worker out of the event loop's+# bounded default pool for the whole handler duration, so a handler's nested+# call queued behind the invocations waiting on it and wedged at exactly the+# pool size. Separately, the local executor kept a single Worker per function+# whose lock was held for the whole handler, so a function invoking *itself*+# deadlocked on the first hop regardless of threading.+#+# Executions now run on a dedicated thread each (core.concurrency.run_reentrant)+# and draw a warm worker from a per-function pool, with AWS's concurrency model+# enforced once, in lambda_svc._acquire_execution_slot.+# ---------------------------------------------------------------------------++_SELF_INVOKE_CODE = """+import json, os, urllib.request++def handler(event, context):+ depth = int(event.get("depth", 0))+ limit = int(event.get("limit", 3))+ if depth >= limit:+ return {"depth": depth, "leaf": True}+ req = urllib.request.Request(+ os.environ["AWS_ENDPOINT_URL"]+ + "/2015-03-31/functions/" + context.function_name + "/invocations",+ data=json.dumps({"depth": depth + 1, "limit": limit}).encode(),+ headers={"Content-Type": "application/json"},+ )+ with urllib.request.urlopen(req, timeout=25) as resp:+ return {"depth": depth, "child": json.loads(resp.read().decode())}+"""+++ os.environ.get("LAMBDA_EXECUTOR", "").lower() == "docker",+ reason="exercises the in-process warm worker pool; the docker executor "+ "spawns a container per concurrent invocation and has no shared lock",+)+def test_lambda_self_invoking_handler_does_not_deadlock(lam):+ """A handler that invokes its own function completes.++ Real AWS allows this and only intervenes at the ~16-hop recursive-loop+ limit. Before the per-function worker pool this deadlocked at the *first*+ hop, with a single caller and no concurrency involved: the nested invoke+ waited on the worker lock its own parent held, and the invocation burned+ its whole timeout.+ """+ name = f"lam-self-invoke-{_uuid_mod.uuid4().hex[:8]}"+ lam.create_function(+ FunctionName=name, Runtime="python3.12", Handler="index.handler",+ Role=_LAMBDA_ROLE, Code={"ZipFile": _make_zip(_SELF_INVOKE_CODE)},+ Timeout=30,+ )+ try:+ resp = lam.invoke(+ FunctionName=name,+ Payload=json.dumps({"depth": 0, "limit": 4}).encode(),+ )+ assert resp.get("FunctionError", "") == "", resp.get("FunctionError")+ payload = json.loads(resp["Payload"].read())+ # Four nested levels, each reporting its own depth, innermost a leaf.+ for expected in range(4):+ assert payload["depth"] == expected+ payload = payload["child"]+ assert payload == {"depth": 4, "leaf": True}+ finally:+ with contextlib.suppress(Exception):+ lam.delete_function(FunctionName=name)+++def test_lambda_reserved_concurrency_throttles_rather_than_queues(lam):+ """At ReservedConcurrentExecutions the extra invocation is throttled.++ AWS returns TooManyRequestsException rather than queueing. Queueing would+ also reintroduce the deadlock: an invocation waiting for a unit its own+ caller holds waits forever.+ """+ import concurrent.futures++ name = f"lam-reserved-{_uuid_mod.uuid4().hex[:8]}"+ code = "import time\ndef handler(event, context):\n time.sleep(1.5)\n return {'ok': True}\n"+ lam.create_function(+ FunctionName=name, Runtime="python3.12", Handler="index.handler",+ Role=_LAMBDA_ROLE, Code={"ZipFile": _make_zip(code)}, Timeout=30,+ )+ try:+ lam.put_function_concurrency(+ FunctionName=name, ReservedConcurrentExecutions=1)++ def _invoke(_i):+ try:+ resp = lam.invoke(FunctionName=name, Payload=b"{}")+ resp["Payload"].read()+ return resp["StatusCode"], "", ""+ except ClientError as exc:+ return (exc.response["ResponseMetadata"]["HTTPStatusCode"],+ exc.response["Error"]["Code"],+ exc.response.get("Reason", ""))++ with concurrent.futures.ThreadPoolExecutor(max_workers=3) as ex:+ results = list(ex.map(_invoke, range(3)))++ throttled = [r for r in results if r[0] == 429]+ succeeded = [r for r in results if r[0] == 200 and not r[1]]+ assert succeeded, f"reservation of 1 must still admit one invoke: {results}"+ assert throttled, (+ f"invocations beyond ReservedConcurrentExecutions=1 must throttle, "+ f"not queue or hang: {results}"+ )+ # The code, not just the status. AWS models exactly one throttle shape+ # for Invoke; anything else is a string AWS never sends, and+ # ``except lambda.exceptions.TooManyRequestsException`` would not catch+ # it — a 429 alone passes on the wrong shape.+ for status, code, reason in throttled:+ assert code == "TooManyRequestsException", (+ f"expected TooManyRequestsException, got {code!r}")+ assert reason == "ReservedFunctionConcurrentInvocationLimitExceeded", (+ f"expected the function-limit Reason, got {reason!r}")+ finally:+ with contextlib.suppress(Exception):+ lam.delete_function(FunctionName=name)# Recursive-loop detection — PutFunctionRecursionConfig made real.## AWS drops an invocation once a request has gone around the same function@@ -10430,3 +10549,90 @@ def test_lambda_inbound_invoke_depth_parsing(headers, expected):from ministack.services.lambda_svc import _inbound_invoke_depthassert _inbound_invoke_depth(headers) == expected+++# ---------------------------------------------------------------------------+# Re-entrancy under concurrency — the rest of the suite issues one request at+# a time, so a service whose blocking work is misclassified passes serially+# and only wedges under load. These fire N callers at once and assert both+# that every caller completes and that the event loop keeps serving.+# ---------------------------------------------------------------------------+def test_concurrent_nested_lambda_invocations_do_not_starve(lam):+ """Callers that each invoke another function must never wedge.++ Three details make this discriminate; without any one of them a build with+ the bug still passes:++ - **distinct caller functions.** One function invoked N times serialises on+ its own worker lock and never reaches the shared pool.+ - **a hold before the nested call**, so every caller is in flight at once+ rather than completing before the next starts.+ - **N above the pool size**, since starvation begins at exactly the pool+ bound.++ A throttle is a correct answer and is allowed — the account concurrency cap+ is deliberately below N on small hosts. What must never happen is a caller+ getting no answer at all, which is what starvation looks like.+ """+ callee = make_probe_lambda(lam, "def handler(event, context):\n return {'pong': True}\n")+ caller_src = f"""+import json, os, time, urllib.request++def handler(event, context):+ time.sleep(float(event.get("hold", 1.5)))+ ep = os.environ["AWS_ENDPOINT_URL"]+ req = urllib.request.Request(+ ep + "/2015-03-31/functions/{callee}/invocations",+ data=json.dumps({{}}).encode(),+ headers={{"Content-Type": "application/json"}},+ )+ return {{"nested": json.loads(urllib.request.urlopen(req, timeout=25).read().decode())}}+"""+ callers = [make_probe_lambda(lam, caller_src, timeout=40) for _ in range(CONCURRENCY_N_NESTED)]++ # A client sized for the burst. The shared fixture client pools far fewer+ # connections than we fire, and the resulting client-side queueing shows up+ # as latency that looks like server stall but is not.+ import boto3+ from botocore.config import Config as _Config++ burst_client = boto3.client(+ "lambda", endpoint_url=ENDPOINT, aws_access_key_id="test",+ aws_secret_access_key="test", region_name=os.environ.get("AWS_DEFAULT_REGION", "us-east-1"),+ config=_Config(retries={"max_attempts": 0}, read_timeout=120,+ max_pool_connections=CONCURRENCY_N_NESTED + 8),+ )++ def invoke(name, hold=1.5):+ try:+ payload = burst_client.invoke(+ FunctionName=name, Payload=json.dumps({"hold": hold}).encode())+ body = payload["Payload"].read().decode() or "{}"+ if payload.get("FunctionError"):+ # A timeout is the starvation signature: the nested call never+ # got a slot. Anything else is the handler's own failure.+ return "wedged" if "timed out" in body.lower() else "error"+ return "ok"+ except Exception as exc:+ return "throttled" if "TooManyRequests" in str(exc) else "wedged"++ # Warm every caller first, in small batches. Otherwise the measured burst+ # pays 70 simultaneous cold starts, which a 2-core CI runner cannot finish+ # inside the function timeout — 16 of 70 timed out that way, which reads as+ # starvation but is only the box running out of CPU. Warm workers make the+ # burst measure scheduling, which is what this test is about.+ for i in range(0, len(callers), 10):+ concurrent_burst(lambda n: invoke(n, hold=0.0), items=callers[i:i + 10])++ with LoopProbe() as probe:+ results = concurrent_burst(invoke, items=callers)+ probe.assert_responsive("nested lambda burst")++ wedged = results.count("wedged")+ assert wedged == 0, (+ f"{wedged}/{CONCURRENCY_N_NESTED} nested invocations never completed — re-entrant work is "+ f"queueing behind itself (ok={results.count('ok')}, "+ f"throttled={results.count('throttled')}, error={results.count('error')})"+ )+ assert results.count("ok") > 0, "no invocation succeeded at all"diff --git a/tests/test_ministack.py b/tests/test_ministack.pyindex d5bbd73f..8cc6fabe 100644--- a/tests/test_ministack.py+++ b/tests/test_ministack.py@@ -733,3 +733,81 @@ class TestExtractS3VhostBucket:def test_bare_s3_without_region(self):assert _extract_s3_vhost_bucket("s3.localhost") is None+++# ---------------------------------------------------------------------------+# Container reaping is scoped to this MiniStack instance+# ---------------------------------------------------------------------------++def _exited_container(daemon, reaper, name, **extra):+ c = daemon.containers.run(name=name, labels=reaper.own_labels("rds", **extra))+ c.status = "exited"+ c.attrs["State"] = {"FinishedAt": ""}+ return c+++def test_reaper_never_touches_another_instances_container(fake_docker, monkeypatch):+ """A second gateway on the same daemon must not reclaim the first's work.++ The reaper reclaimed any exited ministack-labelled container absent from+ *this* process's records. With several instances on one Docker daemon that+ is wrong: `StopDBCluster` leaves an exited container on purpose, expecting a+ later start, and a second gateway has no record of it — so it was reaped+ after the grace period, destroying state its owner meant to keep. Observed+ live by @Areson, one instance removing another's container ~3 minutes after+ it was stopped.+ """+ from ministack.core import container_reaper++ monkeypatch.setattr(container_reaper, "EXITED_GRACE", 0)+ monkeypatch.setenv("GATEWAY_PORT", "4566")+ _exited_container(fake_docker, container_reaper, "owned-by-A", db_id="a")++ monkeypatch.setenv("GATEWAY_PORT", "4599") # a different instance+ monkeypatch.setattr(container_reaper, "_boot_nonce", "nonce-B")+ container_reaper.register_live_ids("rds", lambda: [])++ assert container_reaper.reap_abandoned(fake_docker) == 0+ assert "owned-by-A" in fake_docker.live(), "instance B destroyed instance A's container"+++def test_reaper_still_reclaims_its_own_abandoned_container(fake_docker, monkeypatch):+ """Instance scoping must not cost the reclamation it was added for."""+ from ministack.core import container_reaper++ monkeypatch.setattr(container_reaper, "EXITED_GRACE", 0)+ monkeypatch.setenv("GATEWAY_PORT", "4566")+ monkeypatch.setattr(container_reaper, "_boot_nonce", "nonce-mine")+ _exited_container(fake_docker, container_reaper, "owned-by-me", db_id="mine")+ container_reaper.register_live_ids("rds", lambda: [])++ assert container_reaper.reap_abandoned(fake_docker) == 1+ assert fake_docker.live() == []+++def test_boot_sweep_takes_our_predecessor_but_spares_other_instances(fake_docker, monkeypatch):+ """Boot reclaims the previous run on our port, never another instance.++ Holding the gateway port proves the previous owner of that address is gone,+ so its leftovers are ours to clean; a container labelled for a different+ address belongs to a MiniStack that may still be running.+ """+ from ministack.core import container_reaper++ monkeypatch.setenv("GATEWAY_PORT", "4566")+ monkeypatch.setattr(container_reaper, "_boot_nonce", "run-1")+ _exited_container(fake_docker, container_reaper, "ours-previous-run", db_id="old")++ monkeypatch.setenv("GATEWAY_PORT", "4777")+ monkeypatch.setattr(container_reaper, "_boot_nonce", "other")+ _exited_container(fake_docker, container_reaper, "another-instance", db_id="theirs")++ monkeypatch.setenv("GATEWAY_PORT", "4566") # we restart on our port+ monkeypatch.setattr(container_reaper, "_boot_nonce", "run-2")+ container_reaper.reap_all(fake_docker, include_unlabelled=True)++ assert "ours-previous-run" not in fake_docker.live(), "boot sweep left our own orphan"+ assert "another-instance" in fake_docker.live(), "boot sweep destroyed another instance's container"diff --git a/tests/test_mwaa.py b/tests/test_mwaa.pyindex c1cae46a..adfc799c 100644--- a/tests/test_mwaa.py+++ b/tests/test_mwaa.py@@ -319,13 +319,18 @@ def test_airflow_runtime_identity_and_background_scope_include_region(assert dag_sync_complete.wait(timeout=2)assert docker_client.containers.get_calls == [expected_name, legacy_name]- assert legacy_container.remove_calls == [{"force": True}]+ # v=True reclaims the container's anonymous volume. Verified against+ # docker: named volumes (MWAA's dags/db persistence) are untouched+ # by remove(v=True), so this cannot lose persisted state.+ assert legacy_container.remove_calls == [{"force": True, "v": True}]assert docker_client.containers.run_kwargs["name"] == expected_name- assert docker_client.containers.run_kwargs["labels"] == {+ # Subset: ownership labels (`ministack.instance` / `ministack.boot`)+ # are also present and are not what this test is about.+ assert {"ministack": "mwaa","region": region,"env_name": env_name,- }+ }.items() <= docker_client.containers.run_kwargs["labels"].items()expected_volume_prefix = legacy_name if legacy_volumes else expected_nameassert set(docker_client.containers.run_kwargs["volumes"]) == {f"{expected_volume_prefix}-dags",@@ -356,7 +361,7 @@ def test_reset_stops_containers_in_every_region(monkeypatch):def stop(self, timeout):self.stop_calls.append(timeout)- def remove(self, force):+ def remove(self, force=False, v=False):self.remove_calls.append(force)class FakeContainers:diff --git a/tests/test_opensearch.py b/tests/test_opensearch.pyindex 4225509b..053e38fd 100644--- a/tests/test_opensearch.py+++ b/tests/test_opensearch.py@@ -672,3 +672,44 @@ def test_opensearch_dataplane_names_include_region():)finally:set_request_region(original_region)+++def test_opensearch_cold_image_publishes_the_real_endpoint(fake_docker, monkeypatch):+ """`_Endpoint` is what readers prefer, so the finisher must update it.++ Same defect @Areson reported against ElastiCache, found here by inventory:+ create left `_Endpoint` at the `localhost:BASE_PORT` placeholder and the+ deferred finisher updated only `Endpoint`, so Describe kept reporting an+ address nothing listens on.+ """+ import threading++ from ministack.core.responses import set_request_account_id, set_request_region+ from ministack.services import opensearch as os_++ set_request_account_id("000000000000")+ set_request_region("us-east-1")+ monkeypatch.setattr(os_, "_get_docker", lambda: fake_docker, raising=False)+ monkeypatch.setattr(os_, "_image_is_local", lambda *_a: False, raising=False)+ monkeypatch.setattr(os_, "DATAPLANE_ENABLED", True, raising=False)+ monkeypatch.setattr(os_, "_spawn_dataplane",+ lambda name, ver: ("localhost", 19200, "cid-os", None, None))++ done = threading.Event()+ orig = os_.spawn_background++ def tracked(fn, *a, **kw):+ t = orig(fn, *a, **kw)+ threading.Thread(target=lambda: (t.join(), done.set()), daemon=True).start()+ return t+ monkeypatch.setattr(os_, "spawn_background", tracked)++ os_.create_domain_record({"DomainName": "cold-dom"})+ assert done.wait(timeout=10), "deferred finisher never completed"++ rec = os_._domains["cold-dom"]+ assert rec["_Endpoint"] == "localhost:19200", (+ f"readers prefer _Endpoint and it still says {rec['_Endpoint']!r}"+ )+ assert rec["Endpoint"] == "localhost:19200"diff --git a/tests/test_rds.py b/tests/test_rds.pyindex a8bdaf1b..5cbcee9a 100644--- a/tests/test_rds.py+++ b/tests/test_rds.py@@ -4931,12 +4931,14 @@ def test_rds_restore_state_respawns_docker_container(monkeypatch):assert runs[0]["environment"]["POSTGRES_USER"] == "admin"assert runs[0]["environment"]["POSTGRES_PASSWORD"] == "password123"assert runs[0]["environment"]["POSTGRES_DB"] == "mydb"- assert runs[0]["labels"] == {+ # Subset: ownership labels (`ministack.instance` / `ministack.boot`) are+ # also stamped so the reaper cannot cross instance boundaries.+ assert {"ministack": "rds","db_id": db_id,"account_id": get_account_id(),"region": get_region(),- }+ }.items() <= runs[0]["labels"].items()restored = m._instances.get(db_id)assert restored is not None@@ -5251,13 +5253,22 @@ def test_rds_restore_state_respawns_one_container_per_cluster(assert runs == []release_migration_remove.set()- deadline = time.time() + 2+ # restore_state provisions on a background thread, so everything below+ # depends on that thread having finished. The wait used to be a bare 2 s+ # budget that fell through silently when it expired, and the test then+ # failed on whichever later assertion happened to notice — an empty+ # `removed_volumes` diff, with no hint that the cause was a timeout. It+ # now waits long enough for a loaded CI runner and says so when it gives+ # up, so a slow box reads as slow rather than as a behaviour change.+ deadline = time.time() + 20+ settled = Falsewhile time.time() < deadline:if scenario in ("last-member-deleted-during-readiness","last-members-deleted-before-start",):if m._clusters.get(cluster_id, {}).get("DBClusterMembers") == []:+ settled = Truebreaktime.sleep(0.01)continue@@ -5266,8 +5277,14 @@ def test_rds_restore_state_respawns_one_container_per_cluster(for db_id in ("restored-writer", "restored-reader")}if statuses <= {"available", "failed"}:+ settled = Truebreaktime.sleep(0.01)+ assert settled, (+ f"{scenario}: restore did not settle within 20s — the background "+ f"provisioning thread is still running, so the assertions below "+ f"would report a timeout as a behaviour change"+ )if not writer_removal_succeeds or not reader_removal_succeeds or (scenario == "ownership-mismatch"diff --git a/tests/test_stepfunctions.py b/tests/test_stepfunctions.pyindex e6e951ee..7c539273 100644--- a/tests/test_stepfunctions.py+++ b/tests/test_stepfunctions.py@@ -11,7 +11,7 @@ import boto3import pytestfrom botocore.config import Configfrom botocore.exceptions import ClientError-from conftest import ENDPOINT+from conftest import ENDPOINT, LoopProbe, concurrent_burstdef _make_zip(code: str) -> bytes:@@ -7523,3 +7523,33 @@ def test_sfn_jsonata_format_number(sfn_sync):"neg_sub": "(1,234.50)","plain": "1,234",}+++# ---------------------------------------------------------------------------+# Re-entrancy under concurrency — the rest of the suite issues one request at+# a time, so a service whose blocking work is misclassified passes serially+# and only wedges under load. These fire N callers at once and assert both+# that every caller completes and that the event loop keeps serving.+# ---------------------------------------------------------------------------+def test_concurrent_sync_executions_do_not_block_the_loop(sfn, sfn_sync):+ """StartSyncExecution runs off the loop; N at once must not stall the server."""+ definition = json.dumps({+ "StartAt": "Wait",+ "States": {"Wait": {"Type": "Wait", "Seconds": 1, "Next": "Done"},+ "Done": {"Type": "Pass", "End": True}},+ })+ arn = sfn.create_state_machine(+ name=f"conc-sync-{_uuid_mod.uuid4().hex[:8]}", definition=definition,+ roleArn="arn:aws:iam::000000000000:role/R",+ )["stateMachineArn"]++ def run(i):+ return sfn_sync.start_sync_execution(+ stateMachineArn=arn, input=json.dumps({"i": i}))["status"]++ with LoopProbe() as probe:+ statuses = concurrent_burst(run)+ probe.assert_responsive("sync execution burst")++ assert all(s == "SUCCEEDED" for s in statuses), f"sync executions returned {set(statuses)}"diff --git a/tests/test_transfer.py b/tests/test_transfer.pyindex 0db79888..d62979cf 100644--- a/tests/test_transfer.py+++ b/tests/test_transfer.py@@ -323,7 +323,6 @@ def test_transfer_workos_sftp_workflow(transfer):import asyncioimport osimport socket-import timeimport urllib.requestimport uuid@@ -791,6 +790,11 @@ def test_sftp_auth_uses_server_region_and_stashes_connection_metadata():assert service._sftp_server_state(account_id, west_region, server_id) == "OFFLINE"connection = _FakeConnection()+ # `_MiniStackSSHServer` is built by `_ensure_sftp_runtime()`, not at+ # import — reaching for the class directly is only safe once that has+ # run. Previously this test passed only when some earlier test in the+ # same process happened to trigger it.+ service._ensure_sftp_runtime()auth_server = service._MiniStackSSHServer()auth_server.connection_made(connection)assert auth_server.validate_public_key(