Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

feat(concurrency): route blocking work by whether it can re-enter the server

Nahuel Nucera9bfa0604bbae
diff --git a/ministack/app.py b/ministack/app.py
index edefc34c..29ec527d 100644
--- a/ministack/app.py
+++ b/ministack/app.py
@@ -161,6 +161,8 @@ _NON_S3_VHOST_NAMES = frozenset({
"inspector2", "dsql",
})
+from ministack.core import container_reaper
+from ministack.core.concurrency import spawn_background
from ministack.core.hypercorn_compat import install as _install_hypercorn_compat
from ministack.core.persistence import PERSIST_STATE, load_state, save_all
from ministack.core.responses import _12_DIGIT_RE, set_request_account_id, set_request_region
@@ -231,6 +233,48 @@ def _get_module(name: str):
return mod
+# The loop serving HTTP requests, captured at lifespan startup. Worker threads
+# (Step Functions executions, CloudFormation provisioners) need it to call a
+# service handler, because a handler may genuinely await now.
+_MAIN_LOOP = None
+
+
+def call_service_handler_sync(handler, method, path, headers, body, query_params,
+ timeout: float | None = None):
+ """Call an async service handler from a worker thread and return its result.
+
+ In-process callers used to drive handlers with ``coro.send(None)``, relying
+ on an unwritten invariant that service handlers never actually await. That
+ held only while every handler was synchronous underneath. The moment one
+ awaits — offloading Docker work, for instance — the first step runs with no
+ running event loop and raises ``RuntimeError: no running event loop``, which
+ escapes past the caller's ``except StopIteration``.
+
+ So the coroutine is scheduled on the loop that is actually running and this
+ thread waits for the result. Callers must be on a worker thread: waiting on
+ the loop from the loop itself would deadlock, so that is refused loudly
+ rather than hanging.
+ """
+ loop = _MAIN_LOOP
+ if loop is None or loop.is_closed():
+ # No server running (unit tests, CLI use): drive it on a private loop.
+ return asyncio.run(handler(method, path, headers, body, query_params))
+
+ try:
+ running = asyncio.get_running_loop()
+ except RuntimeError:
+ running = None
+ if running is loop:
+ raise RuntimeError(
+ "call_service_handler_sync must be called from a worker thread, "
+ "not from the event loop — await the handler directly instead"
+ )
+
+ future = asyncio.run_coroutine_threadsafe(
+ handler(method, path, headers, body, query_params), loop)
+ return future.result(timeout)
+
+
def _lazy_handler(module_name: str):
"""Return a callable that lazily imports module_name and delegates to handle_request."""
@@ -1988,6 +2032,8 @@ async def _handle_lifespan(scope, receive, send):
import concurrent.futures
_max_workers = int(os.environ.get("MINISTACK_WORKER_THREADS", "64"))
+ global _MAIN_LOOP
+ _MAIN_LOOP = asyncio.get_running_loop()
asyncio.get_running_loop().set_default_executor(
concurrent.futures.ThreadPoolExecutor(
max_workers=_max_workers,
@@ -2000,7 +2046,28 @@ async def _handle_lifespan(scope, receive, send):
# process. Persistence strips container ids from snapshots, so any
# ministack-labelled container alive at boot is by definition an
# orphan whose name will collide on next create.
- _stop_docker_containers()
+ #
+ # Bounded: the sweep normally finishes in well under a second and we
+ # wait for it, so a create right after boot still sees a clean slate.
+ # But it talks to the Docker daemon, and a slow or wedged daemon must
+ # never stop MiniStack from binding its port — that presents as a
+ # silent hang with an empty log. Past the deadline it finishes in the
+ # background while the server comes up.
+ # Periodic reclamation for the whole process lifetime, not just at
+ # boot: without it, containers (and their anonymous volumes) pile up
+ # for as long as MiniStack runs.
+ container_reaper.start(_reaper_docker_client)
+ # Boot only: also reclaim ownerless leftovers from a MiniStack that
+ # predates the instance labels. Holding the gateway port means any
+ # such container is ours. Shutdown deliberately does not do this —
+ # another instance may be live on this daemon.
+ _reap = spawn_background(_stop_docker_containers, True,
+ thread_name="ministack-boot-reap")
+ _reap.join(timeout=_DOCKER_REAP_BOOT_DEADLINE)
+ if _reap.is_alive():
+ logger.warning(
+ "Docker orphan reap still running after %ss; continuing "
+ "startup without it", _DOCKER_REAP_BOOT_DEADLINE)
if PERSIST_STATE:
_load_persisted_state()
# Start the Transfer Family SFTP listener after persistence is
@@ -2010,14 +2077,21 @@ async def _handle_lifespan(scope, receive, send):
# — its top-level `import asyncssh` pulls cryptography+OpenSSL
# (~2–4 MiB of heap, plus C-level SSL contexts) which is pure
# overhead for callers that aren't using Transfer Family.
+ # Only bind at boot for servers restored from persistence, which a
+ # client can connect to without calling CreateServer first. A fresh
+ # boot binds lazily on CreateServer instead — nobody can connect to
+ # a server that does not exist yet — which keeps asyncssh (and the
+ # cryptography + OpenSSL it drags in, ~26 MB of heap, measured
+ # 39.7 -> 27.9 MiB idle RSS) out of every MiniStack that never
+ # touches Transfer Family. Guarded on the module already being
+ # loaded, mirroring dsql below, so we never import it just to ask.
_sftp_env = os.environ.get("SFTP_ENABLED", "").strip().lower()
+ _transfer_mod = _loaded_modules.get("transfer")
if _sftp_env in ("0", "false", "no", "off"):
logger.debug("SFTP_ENABLED=%s — skipping transfer module import.", _sftp_env)
- else:
+ elif _transfer_mod is not None and _transfer_mod.has_servers():
try:
- from ministack.services import transfer
-
- await transfer.sftp_start()
+ await _transfer_mod.sftp_start()
except Exception as e:
logger.warning("Transfer SFTP startup failed: %s", e)
# Start the IoT mTLS MQTT listener, for the same reason and in the
@@ -2093,36 +2167,43 @@ async def _handle_lifespan(scope, receive, send):
return
-def _stop_docker_containers():
- """Stop all Docker containers managed by MiniStack (RDS, ECS, ElastiCache).
- Uses container labels to find them — does not touch service state.
+# Docker orphan reap bounds. The client timeout caps any single daemon call;
+# the boot deadline caps how long startup will wait for the whole sweep before
+# letting the server bind and finishing the reap in the background.
+_DOCKER_REAP_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))
+_DOCKER_REAP_BOOT_DEADLINE = 10.0
- Skip entirely if no Docker socket is available: importing the docker
- SDK (and its requests/urllib3/idna transitive deps) costs ~1 MiB of
- Python heap before we even know whether there's anything to clean.
- """
+
+def _reaper_docker_client():
+ """Docker client for the periodic reaper, or None when there is no daemon."""
sock = os.environ.get("DOCKER_HOST") or "unix:///var/run/docker.sock"
- if sock.startswith("unix://"):
- sock_path = sock[len("unix://"):]
- if not os.path.exists(sock_path):
- return
+ if sock.startswith("unix://") and not os.path.exists(sock[len("unix://"):]):
+ return None
try:
import docker
- client = docker.from_env()
+ return docker.from_env(timeout=_DOCKER_REAP_TIMEOUT)
except Exception:
+ return None
+
+
+def _stop_docker_containers(include_unlabelled: bool = False):
+ """Remove every MiniStack container. Boot and shutdown only.
+
+ The reclamation rules live in ``core.container_reaper``: this is the
+ process-boundary phase (everything goes), while the periodic pass while the
+ server is live is ownership-aware and much more conservative. Keeping both
+ in one module means one label list and one place to reason about which
+ containers are ours.
+
+ Skips entirely when there is no Docker socket: importing the docker SDK
+ (and its requests/urllib3/idna transitive deps) costs ~1 MiB of Python heap
+ before we even know whether there is anything to clean.
+ """
+ client = _reaper_docker_client()
+ if client is None:
return
- for label in ("ministack=rds", "ministack=ecs", "ministack=elasticache", "ministack=eks", "ministack=lambda", "ministack=dsql"):
- try:
- # all=True so exited-but-not-removed orphans get cleaned at boot.
- for c in client.containers.list(all=True, filters={"label": label}):
- try:
- c.stop(timeout=5)
- c.remove(v=True)
- except Exception:
- pass
- except Exception:
- pass
+ container_reaper.reap_all(client, include_unlabelled=include_unlabelled)
def _build_persistence_save_dict():
diff --git a/ministack/core/concurrency.py b/ministack/core/concurrency.py
new file mode 100644
index 00000000..a04b79e5
--- /dev/null
+++ b/ministack/core/concurrency.py
@@ -0,0 +1,220 @@
+"""Off-loop execution primitives.
+
+MiniStack is a single ASGI process, so every blocking call has to be placed
+deliberately. The rule is not "which service is this" but **can this work
+re-enter the server**:
+
+- Work that cannot re-enter (an outbound HTTP call to a real endpoint, a
+ DuckDB query, a zip extraction) is safe on a bounded shared pool. Use
+ :func:`run_offloop`.
+- Work that CAN re-enter (anything that runs a Lambda handler, a Docker
+ container that talks back to 4566, a CloudFormation custom resource
+ blocking on its ResponseURL callback) must never take a slot from a bounded
+ pool: the nested request needs a slot of its own, and it queues behind the
+ very call waiting on it. Deadlock arrives at exactly the pool size. Use
+ :func:`run_reentrant`, which gives each such call its own thread.
+- Fire-and-forget work (delivery loops, reapers, background provisioning) uses
+ :func:`spawn_background` so it is named, its exceptions are logged, and it is
+ visible to :func:`active_counts` instead of being an anonymous
+ ``threading.Thread``.
+
+Nothing blocking belongs on the event loop itself. A blocking call there
+freezes every service at once, which is strictly worse than exhausting a pool.
+
+Bounding in-flight re-entrant work is a *policy* question that belongs to the
+caller (Lambda enforces AWS's concurrency model and throttles), not to this
+module: a blocking gate here would re-create the deadlock it exists to remove.
+
+Moving handlers off the loop also removed the serialisation they were silently
+relying on. Under the loop, read-modify-write on a service record could not
+interleave; off it, every ``if record still exists: publish into it`` became a
+race, and a delete landing in the gap leaves a container nothing references. Use
+:func:`resource_lock` for those, under one rule: **hold it across a record
+mutation, never across a Docker call or a re-entrant dispatch.** A lock held
+across either is how #1277's per-service locking deadlocked.
+"""
+
+import asyncio
+import contextvars
+import logging
+import threading
+
+logger = logging.getLogger("concurrency")
+
+# Live counters, for diagnostics via /_ministack/health.
+_counts_lock = threading.Lock()
+_reentrant_threads = 0
+_background_threads = 0
+
+
+def active_counts() -> dict:
+ """Snapshot of threads this module currently has running."""
+ with _counts_lock:
+ return {"reentrant": _reentrant_threads, "background": _background_threads}
+
+
+_resource_locks: dict = {}
+_resource_locks_guard = threading.Lock()
+
+
+def resource_lock(scope: str, key: str) -> threading.Lock:
+ """A lock for one resource — e.g. ``resource_lock("elasticache", cluster_id)``.
+
+ For making a read-modify-write on a service record atomic now that handlers
+ no longer run serialised on the event loop. Typical use is a deferred
+ provisioner publishing what it started, versus a delete removing the record:
+
+ with resource_lock("elasticache", cluster_id): # short, no I/O
+ rec = _clusters.get(cluster_id)
+ stale = rec is not record or rec.get("_deleting")
+ if not stale:
+ rec["_docker_container_id"] = container_id
+ if stale:
+ tear_down(container_id) # outside the lock
+
+ Two rules keep this safe:
+
+ - **Never hold it across a Docker call or anything that can re-enter
+ MiniStack.** That is what makes per-service locking deadlock: a container
+ MiniStack starts is handed ``AWS_ENDPOINT_URL`` pointing back at MiniStack,
+ so it calls in while the starter still holds the lock.
+ - **Compare record identity, not truthiness.** ``if rec:`` cannot tell a
+ surviving record from a same-named one created after a delete; ``rec is
+ record`` can.
+
+ Locks are kept for the process lifetime. They are one mutex per resource id
+ ever seen, which is far cheaper than the bookkeeping to reclaim them safely.
+ """
+ with _resource_locks_guard:
+ return _resource_locks.setdefault((scope, key), threading.Lock())
+
+
+def _bump(kind: str, delta: int) -> None:
+ global _reentrant_threads, _background_threads
+ with _counts_lock:
+ if kind == "reentrant":
+ _reentrant_threads += delta
+ else:
+ _background_threads += delta
+
+
+async def run_offloop(fn, *args, **kwargs):
+ """Run blocking work that cannot re-enter the server, on the shared pool.
+
+ Thin alias for :func:`asyncio.to_thread` so call sites state their
+ classification explicitly and an audit can distinguish "reviewed, cannot
+ re-enter" from "nobody thought about it".
+ """
+ return await asyncio.to_thread(fn, *args, **kwargs)
+
+
+async def run_reentrant(fn, *args, thread_name: str = "ministack-reentrant"):
+ """Run blocking work that may call back into this server, on its own thread.
+
+ A dedicated thread per call, not a pooled worker: nested requests draw from
+ the same pool, so a bound of any size only moves the deadlock edge rather
+ than removing it.
+
+ ContextVars propagate as ``asyncio.to_thread`` would.
+
+ Cancellation defers to completion. A thread cannot be cancelled once it is
+ running, so if a disconnected client let the awaiter unwind immediately, the
+ caller would proceed — releasing a lease, taking a reset barrier, tearing
+ down request state — while the worker is still mutating that same state. So
+ the wait is shielded, repeated cancellation is absorbed until the worker
+ finishes, and the original ``CancelledError`` is re-raised afterwards. The
+ awaiting task is suspended, not the loop, so this costs nothing but the
+ task's own latency.
+ """
+ loop = asyncio.get_running_loop()
+ future: asyncio.Future = loop.create_future()
+ ctx = contextvars.copy_context()
+
+ def _deliver(result, exc):
+ if not future.done():
+ if exc is not None:
+ future.set_exception(exc)
+ else:
+ future.set_result(result)
+
+ def _post(result, exc):
+ try:
+ loop.call_soon_threadsafe(_deliver, result, exc)
+ except RuntimeError:
+ pass # loop already closed (shutdown) — nobody left to deliver to
+
+ def _run():
+ result, exc = None, None
+ try:
+ result = ctx.run(fn, *args)
+ except Exception as e: # noqa: BLE001 — delivered to the awaiter
+ exc = e
+ except BaseException as e:
+ # SystemExit / KeyboardInterrupt: hand it to the awaiter so the
+ # request fails instead of hanging on a future nothing resolves,
+ # then let it unwind this thread rather than swallowing it.
+ _post(None, e)
+ _bump("reentrant", -1)
+ raise
+ _post(result, exc)
+ _bump("reentrant", -1)
+
+ _bump("reentrant", 1)
+ try:
+ threading.Thread(target=_run, daemon=True, name=thread_name).start()
+ except RuntimeError as exc:
+ _bump("reentrant", -1)
+ # OS thread limit. Fail the request loudly instead of awaiting a future
+ # nothing will ever resolve.
+ raise RuntimeError(f"cannot start {thread_name} thread: {exc}") from exc
+
+ try:
+ return await asyncio.shield(future)
+ except asyncio.CancelledError as cancellation:
+ while not future.done():
+ try:
+ await asyncio.shield(future)
+ except asyncio.CancelledError:
+ continue # absorb repeated cancellation; the worker is still live
+ except Exception:
+ break # the worker failed; that is a completion too
+ # Consume a failed worker's exception so asyncio does not report it as
+ # never-retrieved after the awaiting request has already gone away.
+ if future.done() and not future.cancelled():
+ try:
+ future.result()
+ except Exception:
+ pass
+ raise cancellation
+
+
+def spawn_background(fn, *args, thread_name: str = "ministack-background", **kwargs):
+ """Start named fire-and-forget work with its exceptions logged.
+
+ Replaces bare ``threading.Thread(target=..., daemon=True).start()`` so that
+ background failures surface in the log instead of dying silently, and so
+ the thread is countable.
+ """
+ ctx = contextvars.copy_context()
+
+ def _run():
+ try:
+ ctx.run(lambda: fn(*args, **kwargs))
+ except Exception:
+ logger.exception("background task %s failed", thread_name)
+ finally:
+ _bump("background", -1)
+
+ _bump("background", 1)
+ thread = threading.Thread(target=_run, daemon=True, name=thread_name)
+ try:
+ thread.start()
+ except RuntimeError:
+ # OS thread limit. `_run` never executes, so its `finally` never runs
+ # and the counter would leak — permanently, and precisely when someone
+ # is reading it to diagnose thread exhaustion. Callers that can fall
+ # back to running inline (firehose's Iceberg delivery) catch this, so
+ # it must still propagate. `run_reentrant` does the same.
+ _bump("background", -1)
+ raise
+ return thread
diff --git a/ministack/core/container_reaper.py b/ministack/core/container_reaper.py
new file mode 100644
index 00000000..2a91682b
--- /dev/null
+++ b/ministack/core/container_reaper.py
@@ -0,0 +1,281 @@
+"""Periodic reclamation of MiniStack's Docker containers.
+
+Without this, a long-running MiniStack accumulates containers — and the
+anonymous volumes they carry — for its whole lifetime. Only ECS had a reaper;
+RDS, EKS, ElastiCache, MWAA and OpenSearch containers survived until the process
+restarted, and the boot sweep then paid for all of them at once (measured: 119s
+on a healthy daemon with a session's worth of orphans).
+
+The rule is deliberately conservative, because a *stopped* container is not
+necessarily garbage: ``StopDBInstance`` leaves an exited container that
+``StartDBInstance`` must be able to restart. So a container is only reclaimed
+when it is unambiguously garbage:
+
+- state is ``created`` or ``dead`` — never actually ran, or the daemon lost it;
+- or it is ``exited`` **and** no live service record still references its id.
+
+Services publish the ids they still own via ``register_live_ids``. A service
+that does not register is never reaped from here — silence means "unknown", not
+"garbage".
+
+Reaping is also scoped to *this* MiniStack. Several instances can share a Docker
+daemon, and "not in my records" says nothing about another instance's containers:
+a ``StopDBCluster``-ed database is exited by design and expected to restart, so a
+second gateway would delete it after the grace period. Every container carries
+``ministack.instance`` — the gateway address, which only one process can hold, so
+it is an identity rather than a guess — and ``ministack.boot``, a per-process
+nonce. The periodic pass matches both, so it cannot see another instance's
+containers at all; the boot sweep matches the address only, since holding the
+port proves the previous owner is gone.
+"""
+
+import concurrent.futures
+import logging
+import os
+import socket
+import threading
+import time
+import uuid
+
+logger = logging.getLogger("container_reaper")
+
+REAP_INTERVAL = 60.0
+# An exited container gets this long before it is considered abandoned, so a
+# service that is mid-restart is not raced.
+EXITED_GRACE = 120.0
+
+# Label carrying which MiniStack owns a container, and which run of it.
+INSTANCE_LABEL = "ministack.instance"
+BOOT_LABEL = "ministack.boot"
+
+_providers: dict = {}
+_lock = threading.Lock()
+_started = False
+_boot_nonce = uuid.uuid4().hex[:12]
+
+
+def instance_id() -> str:
+ """Identity of this MiniStack: the address it serves on.
+
+ The gateway port is the one thing that is provably exclusive — two instances
+ cannot bind it — so it identifies an instance without any coordination.
+ """
+ host = os.environ.get("MINISTACK_HOSTNAME") or socket.gethostname()
+ port = os.environ.get("GATEWAY_PORT") or os.environ.get("EDGE_PORT") or "4566"
+ return f"{host}:{port}"
+
+
+def own_labels(service: str, **extra) -> dict:
+ """Labels every MiniStack-created container must carry.
+
+ ``service`` is the value of the ``ministack`` label (e.g. "rds"). Anything in
+ ``extra`` is merged, so callers keep their own labels.
+ """
+ labels = {
+ "ministack": service,
+ INSTANCE_LABEL: instance_id(),
+ BOOT_LABEL: _boot_nonce,
+ }
+ labels.update({k: v for k, v in extra.items() if v is not None})
+ return labels
+
+
+def register_live_ids(label: str, provider) -> None:
+ """Register a callable returning the container ids ``label`` still owns.
+
+ ``label`` is the value of the container's ``ministack`` label (e.g. "rds").
+ The provider must return an iterable of container ids; anything it omits
+ with an exited container is treated as abandoned.
+ """
+ with _lock:
+ _providers[label] = provider
+
+
+def _live_ids() -> tuple[set, set]:
+ """(ids still owned, labels that answered) — labels that don't answer are skipped."""
+ live, known = set(), set()
+ with _lock:
+ providers = dict(_providers)
+ for label, provider in providers.items():
+ try:
+ ids = provider() or ()
+ except Exception as exc:
+ logger.debug("reaper: provider for %s failed: %s", label, exc)
+ continue
+ known.add(label)
+ live.update(i for i in ids if i)
+ return live, known
+
+
+def reap_abandoned(docker_client) -> int:
+ """Remove abandoned containers. Returns how many were reclaimed."""
+ if docker_client is None:
+ return 0
+ live, known = _live_ids()
+ now = time.time()
+ removed = 0
+ try:
+ # Scoped to this instance *and* this run of it. Another MiniStack's
+ # containers are not merely skipped later — they are never listed, so no
+ # amount of downstream logic can reclaim them. Containers predating the
+ # labels are likewise invisible here; the boot sweep still gets them.
+ containers = docker_client.containers.list(all=True, filters={"label": [
+ f"{INSTANCE_LABEL}={instance_id()}",
+ f"{BOOT_LABEL}={_boot_nonce}",
+ ]})
+ except Exception as exc:
+ logger.debug("reaper: listing containers failed: %s", exc)
+ return 0
+
+ for c in containers:
+ try:
+ label = (c.labels or {}).get("ministack", "")
+ status = c.status
+ if status not in ("created", "dead", "exited"):
+ continue # running / restarting / paused: not ours to judge
+ if label not in known:
+ # No provider for this service, so nothing can vouch for the
+ # container. Silence means "unknown", not "garbage" — a service
+ # that creates a container and starts it a moment later (ECS
+ # tasks pass through `created`) must never be raced.
+ continue
+ if c.id in live:
+ continue # a stopped-but-live resource (e.g. StopDBInstance)
+ finished = (c.attrs.get("State", {}) or {}).get("FinishedAt", "")
+ if finished and _age_seconds(finished, now) < EXITED_GRACE:
+ continue
+ if status == "created":
+ # `created` has no FinishedAt; use Created to age it out so a
+ # container mid-start is never taken from under its service.
+ created_at = c.attrs.get("Created", "")
+ if created_at and _age_seconds(created_at, now) < EXITED_GRACE:
+ continue
+ c.remove(force=True, v=True) # v=True: reclaim the anonymous volume too
+ removed += 1
+ except Exception:
+ continue
+ if removed:
+ logger.info("Reaped %d abandoned container(s)", removed)
+ return removed
+
+
+def _age_seconds(finished_at: str, now: float) -> float:
+ """Seconds since a Docker RFC3339 timestamp; 0 if it cannot be parsed."""
+ import datetime
+ try:
+ ts = finished_at.replace("Z", "+00:00")
+ if "." in ts:
+ head, rest = ts.split(".", 1)
+ frac, _, tz = rest.partition("+")
+ ts = f"{head}.{frac[:6]}+{tz}" if tz else f"{head}.{frac[:6]}"
+ return max(0.0, now - datetime.datetime.fromisoformat(ts).timestamp())
+ except Exception:
+ return 0.0
+
+
+def start(get_docker) -> None:
+ """Start the reaper thread once. ``get_docker`` is called each pass."""
+ global _started
+ with _lock:
+ if _started:
+ return
+ _started = True
+
+ def _loop():
+ while True:
+ time.sleep(REAP_INTERVAL)
+ try:
+ reap_abandoned(get_docker())
+ except Exception as exc:
+ logger.debug("reaper iteration failed: %s", exc)
+
+ threading.Thread(target=_loop, daemon=True, name="ministack-container-reaper").start()
+
+# Every label a service stamps on a container, plus the key-only backstops so a
+# new service (or an older container predating its label) is still reclaimed.
+# One list, one place: the boot/shutdown sweep and the periodic pass both use it.
+SERVICE_LABELS = (
+ "ministack=rds", "ministack=ecs", "ministack=elasticache", "ministack=eks",
+ "ministack=lambda", "ministack=dsql", "ministack=mwaa", "ministack=glue",
+ "ministack=codebuild", "ministack=opensearch",
+ "ministack", "com.ministack.service",
+)
+
+
+def reap_all(docker_client, stop_timeout: int = 2, include_unlabelled: bool = False) -> int:
+ """Remove every MiniStack container, whatever its state.
+
+ For process boundaries only — boot and shutdown. At boot any surviving
+ container **of ours** is by definition an orphan of a dead process: we hold
+ the gateway port now, so whichever process created them is gone (persistence
+ strips container ids from snapshots, so nothing can still own one). At
+ shutdown everything is going away regardless. Neither assumption holds while
+ the process is live, which is why :func:`reap_abandoned` exists separately
+ and is far more conservative.
+
+ Scoped by ``ministack.instance`` but *not* by boot nonce — the whole point is
+ to reclaim a previous run's containers, which carry a different nonce.
+
+ A container owned by a *different* instance is never reclaimed here, at boot
+ or at shutdown. Shutdown is not a licence to sweep broadly: another MiniStack
+ may be running on the same daemon, and taking its containers down as we exit
+ is the very failure this scoping exists to prevent.
+
+ ``include_unlabelled`` additionally matches containers that carry a MiniStack
+ service label but *no* instance label — those predate this scheme and cannot
+ be attributed to anyone. Boot uses it, because holding the gateway port means
+ any such leftover is ours to clean; shutdown does not.
+ """
+ if docker_client is None:
+ return 0
+ mine = instance_id()
+ seen, targets = set(), []
+
+ def _collect(selector, accept):
+ try:
+ found = docker_client.containers.list(all=True, filters={"label": selector})
+ except Exception as exc:
+ logger.debug("reap_all: listing %s failed: %s", selector, exc)
+ return
+ for c in found:
+ if c.id in seen or not accept(c):
+ continue
+ seen.add(c.id)
+ targets.append(c)
+
+ _collect([f"{INSTANCE_LABEL}={mine}"], lambda c: True)
+ if include_unlabelled:
+ for service_label in SERVICE_LABELS:
+ # Only ownerless leftovers. A container carrying someone else's
+ # instance label is off limits however it was found.
+ _collect([service_label], lambda c: not (c.labels or {}).get(INSTANCE_LABEL))
+ return drop_containers(targets, stop_timeout=stop_timeout)
+
+
+def drop_containers(containers, stop_timeout: int = 2, force: bool = False) -> int:
+ """Stop and remove containers concurrently. Returns how many went away.
+
+ Concurrent because ``stop`` is a per-container round trip that blocks until
+ the container actually dies — serially that is O(n) seconds of wall clock.
+ It matters because this runs on the ``/_ministack/reset`` path while the
+ reset lock is held, so every other request queues behind it. Measured: a
+ run that left 30 running ECS tasks took 54.4s serially, past the 45s the
+ caller was willing to wait. The daemon parallelises stops fine; the cap only
+ keeps a huge sweep from opening hundreds of sockets at once.
+ """
+ targets = [c for c in containers if c is not None]
+ if not targets:
+ return 0
+
+ def _drop(c):
+ try:
+ c.stop(timeout=stop_timeout)
+ c.remove(v=True, force=force) # v=True: reclaim the anonymous volume too
+ return True
+ except Exception:
+ return False
+
+ with concurrent.futures.ThreadPoolExecutor(
+ max_workers=min(16, len(targets)), thread_name_prefix="ministack-reap") as pool:
+ return sum(1 for ok in pool.map(_drop, targets) if ok)
+
diff --git a/ministack/core/lambda_runtime.py b/ministack/core/lambda_runtime.py
index 113bb7f0..7559068f 100644
--- a/ministack/core/lambda_runtime.py
+++ b/ministack/core/lambda_runtime.py
@@ -76,6 +76,18 @@ def _account_region_from_function_config(config: dict) -> tuple[str, str]:
_workers: dict = {}
_lock = threading.Lock()
+# No per-function ceiling on warm subprocesses: AWS lets an unreserved function
+# scale into the account pool, and capping width here would invent a throttle
+# AWS never sends and diverge from the docker executor, which spawns freely.
+# Concurrency is bounded once, at lambda_svc._acquire_execution_slot, via
+# ReservedConcurrentExecutions and the existing LAMBDA_ACCOUNT_CONCURRENCY.
+# The reaper reclaims the surplus subprocesses a burst leaves behind.
+_LOCAL_MAX_WORKERS = 0
+# Seconds a surplus worker may sit idle before it is reaped (the first worker
+# per function is never reaped, so warm starts are unaffected). Shares the knob
+# that already governs warm-container eviction rather than adding a second one.
+_LOCAL_WORKER_TTL = float(os.environ.get("LAMBDA_WARM_TTL_SECONDS", "300"))
+
# ---------------------------------------------------------------------------
# Recursive-loop lineage (runs inside every ministack-owned Python child)
# ---------------------------------------------------------------------------
@@ -935,6 +947,10 @@ class Worker:
self._proc = None
self._tmpdir = None
self._lock = threading.Lock()
+ # Leased out to an in-flight invocation. Guarded by the module ``_lock``
+ # (see acquire_worker/release_worker), not by ``self._lock``.
+ self.in_use = False
+ self.last_used = time.time()
self._cold = True
self._start_time = None
self._stderr_queue: queue.Queue = queue.Queue()
@@ -1243,9 +1259,9 @@ class Worker:
if reader.is_alive():
# Timeout — kill the worker process
logger.warning("Lambda %s timed out after %ds — killing worker", self.func_name, timeout)
- if self._proc:
- self._proc.kill()
- self._proc = None
+ proc, self._proc = self._proc, None
+ _signal(proc)
+ _collect(proc, time.monotonic() + _REAP_GRACE)
return {
"status": "error",
"error": f"Task timed out after {timeout}.00 seconds",
@@ -1259,37 +1275,195 @@ class Worker:
response = result_box[0]
if response.get("status") == "error":
- if self._proc and self._proc.poll() is None:
- self._proc.terminate()
- self._proc = None
+ proc, self._proc = self._proc, None
+ _signal(proc)
+ _collect(proc, time.monotonic() + _REAP_GRACE)
response["cold_start"] = cold
# Bounded drain — replaces the fixed 50ms sleep that was paid
# by every warm invocation. Typical completion is 1–10ms.
response["log"] = self._drain_stderr_bounded()
return response
- def kill(self):
- if self._proc and self._proc.poll() is None:
- self._proc.terminate()
- self._proc = None
+ def _discard_tmpdir(self):
if self._tmpdir and os.path.exists(self._tmpdir):
shutil.rmtree(self._tmpdir, ignore_errors=True)
+ def kill(self):
+ kill_workers([self])
-def get_or_create_worker(func_name: str, config: dict, code_zip: bytes,
- qualifier: str = "$LATEST") -> Worker:
+
+# Seconds a worker gets to exit on SIGTERM before it is SIGKILLed. Paid once per
+# *batch*, never once per worker — see kill_workers.
+_REAP_GRACE = 2.0
+
+
+def _signal(proc) -> None:
+ """Ask a worker subprocess to exit. Does not wait."""
+ if proc is None:
+ return
+ try:
+ if proc.poll() is None:
+ proc.terminate()
+ except Exception:
+ pass
+
+
+def _collect(proc, deadline: float) -> None:
+ """Wait for a signalled subprocess and reap it, escalating if it ignores SIGTERM.
+
+ Collecting is the point. Signalling alone leaves a zombie: ``subprocess``
+ only reaps an abandoned Popen lazily, when the *next* Popen is constructed,
+ so an **idle** server keeps every zombie it made. On a host that is invisible
+ — the count is small and something else is always spawning — but in a
+ container MiniStack is PID 1: the zombies are its own children, nothing else
+ will ever collect them, and they hold PID-table slots against the container's
+ pid limit. Measured: a 320-invocation burst left 244 zombies that survived
+ indefinitely while the server sat idle.
+
+ Warm-worker pooling is what made this reachable. One worker per function was
+ killed rarely; a pool of them is reaped by the hundred.
+ """
+ if proc is None:
+ return
+ try:
+ proc.wait(timeout=max(0.0, deadline - time.monotonic()))
+ except Exception:
+ try:
+ proc.kill()
+ proc.wait(timeout=0.5)
+ except Exception:
+ pass # unkillable: leave it rather than block the caller
+
+
+def kill_workers(workers) -> None:
+ """Terminate and reap a batch of workers, paying the grace period once.
+
+ Two-phase on purpose. Signalling every worker first and only then waiting
+ makes a mass teardown cost ``_REAP_GRACE`` in total; doing it worker by
+ worker would cost ``n * _REAP_GRACE``, which on a pool built by a burst is
+ minutes — and ``reset()`` used to pay it holding the global worker lock.
+ """
+ workers = [w for w in workers if w is not None]
+ if not workers:
+ return
+ procs = []
+ for worker in workers:
+ proc, worker._proc = worker._proc, None
+ procs.append(proc)
+ _signal(proc)
+ deadline = time.monotonic() + _REAP_GRACE
+ for proc in procs:
+ _collect(proc, deadline)
+ for worker in workers:
+ try:
+ worker._discard_tmpdir()
+ except Exception:
+ pass
+
+
+def _worker_key(func_name: str, config: dict, qualifier: str) -> str:
# Include account ID and region in the key to isolate workers across
# accounts and regions. Two regions deploying the same function name must
# not share a worker.
account, region = _account_region_from_function_config(config)
- key = f"{account}:{region}:{func_name}:{qualifier}"
+ return f"{account}:{region}:{func_name}:{qualifier}"
+
+
+def acquire_worker(func_name: str, config: dict, code_zip: bytes,
+ qualifier: str = "$LATEST", max_concurrency: int | None = None):
+ """Lease a free execution environment for this function.
+
+ Each key holds a *pool* of workers, mirroring how AWS gives concurrent
+ invocations separate execution environments (and how the Docker executor's
+ ``_warm_pool`` already behaves). A single shared worker per function could
+ not: ``Worker.invoke`` holds the worker's lock for the whole handler, so a
+ handler that invokes its own function — which real AWS supports, up to the
+ ~16-hop recursive-loop limit — waited on the lock its own caller held and
+ deadlocked. The thread the invocation runs on is irrelevant to that; the
+ lock is the bottleneck.
+
+ ``max_concurrency`` is an optional *resource* ceiling on how many warm
+ subprocesses may exist for one function, not a concurrency limit — AWS
+ concurrency is enforced upstream in ``lambda_svc._acquire_execution_slot``,
+ once, for every executor. It defaults to unbounded, matching the docker
+ executor. At a ceiling the caller gets ``(None, "func_cap")`` rather than a
+ blocking wait, because waiting for a worker held by your own caller is the
+ deadlock this exists to remove.
+
+ Returns ``(worker, "reused"|"spawn")`` or ``(None, "func_cap")``. Release
+ with :func:`release_worker`.
+ """
+ key = _worker_key(func_name, config, qualifier)
+ cap = max_concurrency or _LOCAL_MAX_WORKERS
with _lock:
- worker = _workers.get(key)
- if worker is not None:
- return worker
+ entries = _workers.setdefault(key, [])
+ for worker in entries:
+ if not worker.in_use:
+ worker.in_use = True
+ worker.last_used = time.time()
+ return worker, "reused"
+ if cap and len(entries) >= cap:
+ return None, "func_cap"
worker = Worker(func_name, config, code_zip)
- _workers[key] = worker
- return worker
+ worker.in_use = True
+ worker.last_used = time.time()
+ entries.append(worker)
+ return worker, "spawn"
+
+
+def reap_idle_workers(ttl: float = None) -> int:
+ """Kill workers idle longer than ``ttl`` and drop them from their pool.
+
+ Without this the pools only ever grow: a burst of concurrency leaves its
+ extra subprocesses alive forever. The first worker of each key is kept so
+ the common warm-start path is unaffected.
+ """
+ ttl = _LOCAL_WORKER_TTL if ttl is None else ttl
+ now = time.time()
+ killed = []
+ with _lock:
+ for key, entries in list(_workers.items()):
+ keep = []
+ for i, worker in enumerate(entries):
+ idle = now - getattr(worker, "last_used", now)
+ if i == 0 or worker.in_use or idle < ttl:
+ keep.append(worker)
+ else:
+ killed.append(worker)
+ if keep:
+ _workers[key] = keep
+ else:
+ _workers.pop(key, None)
+ kill_workers(killed)
+ return len(killed)
+
+
+def release_worker(worker: Worker) -> None:
+ """Return a leased worker to its pool."""
+ if worker is None:
+ return
+ with _lock:
+ worker.in_use = False
+
+
+def workers_in_use() -> int:
+ """Count of leased workers across every function, for concurrency limits."""
+ with _lock:
+ return sum(1 for lst in _workers.values() for w in lst if w.in_use)
+
+
+def get_or_create_worker(func_name: str, config: dict, code_zip: bytes,
+ qualifier: str = "$LATEST") -> Worker:
+ """Back-compat lease that never reports a cap and is released immediately.
+
+ Retained for callers that hold a worker only for the duration of a single
+ ``invoke`` on the calling thread. The lease is dropped before returning, so
+ such callers still serialize on ``Worker._lock`` exactly as before rather
+ than silently retaining a pool slot they never release.
+ """
+ worker, _reason = acquire_worker(func_name, config, code_zip, qualifier=qualifier)
+ release_worker(worker)
+ return worker
def invalidate_worker(func_name: str, qualifier: str = None,
@@ -1324,16 +1498,16 @@ def invalidate_worker(func_name: str, qualifier: str = None,
return True
with _lock:
- to_remove = [k for k in _workers if _matches(k)]
- for k in to_remove:
- worker = _workers.pop(k, None)
- if worker:
- worker.kill()
+ doomed = [w for k in [k for k in _workers if _matches(k)]
+ for w in (_workers.pop(k, []) or [])]
+ # Outside the lock: terminating a batch takes as long as the slowest worker
+ # takes to exit, and holding the pool lock for that blocks every invocation.
+ kill_workers(doomed)
def reset():
"""Terminate all warm workers, clean up temp dirs, and clear the pool."""
with _lock:
- for worker in list(_workers.values()):
- worker.kill()
+ doomed = [w for entries in _workers.values() for w in entries]
_workers.clear()
+ kill_workers(doomed)
diff --git a/ministack/core/responses.py b/ministack/core/responses.py
index a706f6da..e81449e4 100644
--- a/ministack/core/responses.py
+++ b/ministack/core/responses.py
@@ -131,13 +131,15 @@ class AccountScopedDict:
return self._scoped(key) in self._data
def __len__(self):
- return sum(1 for k in self._data if self._is_mine(k))
+ # Snapshot: a concurrent insert during a live walk raises
+ # RuntimeError("dictionary changed size during iteration").
+ return sum(1 for k in list(self._data) if self._is_mine(k))
def __bool__(self):
- return any(self._is_mine(k) for k in self._data)
+ return any(self._is_mine(k) for k in list(self._data))
def __iter__(self):
- for k in self._data:
+ for k in list(self._data):
if self._is_mine(k):
yield self._unscope(k)
@@ -163,13 +165,13 @@ class AccountScopedDict:
return self._data.setdefault(self._scoped(key), default)
def keys(self):
- return [self._unscope(k) for k in self._data if self._is_mine(k)]
+ return [self._unscope(k) for k in list(self._data) if self._is_mine(k)]
def values(self):
- return [v for k, v in self._data.items() if self._is_mine(k)]
+ return [v for k, v in list(self._data.items()) if self._is_mine(k)]
def items(self):
- return [(self._unscope(k), v) for k, v in self._data.items() if self._is_mine(k)]
+ return [(self._unscope(k), v) for k, v in list(self._data.items()) if self._is_mine(k)]
def update(self, other):
if isinstance(other, AccountScopedDict):
@@ -241,13 +243,15 @@ class AccountRegionScopedDict:
return self._scoped(key) in self._data
def __len__(self):
- return sum(1 for k in self._data if self._is_mine(k))
+ # Snapshot: a concurrent insert during a live walk raises
+ # RuntimeError("dictionary changed size during iteration").
+ return sum(1 for k in list(self._data) if self._is_mine(k))
def __bool__(self):
- return any(self._is_mine(k) for k in self._data)
+ return any(self._is_mine(k) for k in list(self._data))
def __iter__(self):
- for k in self._data:
+ for k in list(self._data):
if self._is_mine(k):
yield self._unscope(k)
@@ -273,19 +277,19 @@ class AccountRegionScopedDict:
return self._data.setdefault(self._scoped(key), default)
def keys(self):
- return [self._unscope(k) for k in self._data if self._is_mine(k)]
+ return [self._unscope(k) for k in list(self._data) if self._is_mine(k)]
def values(self):
- return [v for k, v in self._data.items() if self._is_mine(k)]
+ return [v for k, v in list(self._data.items()) if self._is_mine(k)]
def items(self):
- return [(self._unscope(k), v) for k, v in self._data.items() if self._is_mine(k)]
+ return [(self._unscope(k), v) for k, v in list(self._data.items()) if self._is_mine(k)]
def values_scoped(self, account_id, region):
- return [v for k, v in self._data.items() if k[:2] == (account_id, region)]
+ return [v for k, v in list(self._data.items()) if k[:2] == (account_id, region)]
def items_scoped(self, account_id, region):
- return [(self._unscope(k), v) for k, v in self._data.items() if k[:2] == (account_id, region)]
+ return [(self._unscope(k), v) for k, v in list(self._data.items()) if k[:2] == (account_id, region)]
def all_values(self):
return list(self._data.values())
diff --git a/ministack/services/alb.py b/ministack/services/alb.py
index 8a796c93..8032a31a 100644
--- a/ministack/services/alb.py
+++ b/ministack/services/alb.py
@@ -31,7 +31,8 @@ import time
from urllib.parse import parse_qs
from ministack.core.arn import ArnParseError, parse_arn
-from ministack.core.persistence import PERSIST_STATE, load_state
+from ministack.core.concurrency import run_reentrant
+from ministack.core.persistence import load_state
from ministack.core.responses import AccountRegionScopedDict, AccountScopedDict, get_account_id, get_region, new_uuid
logger = logging.getLogger("alb")
@@ -1268,7 +1269,8 @@ async def _invoke_lambda_target(function_ref, tg_arn, method, path, headers, bod
)
return invocation_result
- exec_result = await asyncio.to_thread(_invoke_and_record_metrics)
+ exec_result = await run_reentrant(_invoke_and_record_metrics,
+ thread_name="ministack-alb-invoke")
lambda_response, _err = lambda_svc.lambda_execute_result_to_api_proxy_response(exec_result)
if exec_result.get("error"):
diff --git a/ministack/services/apigateway.py b/ministack/services/apigateway.py
index 28e013ac..948efe7f 100644
--- a/ministack/services/apigateway.py
+++ b/ministack/services/apigateway.py
@@ -52,10 +52,10 @@ import urllib.parse
import urllib.request
from ministack.core.arn import ArnParseError, parse_arn
+from ministack.core.concurrency import run_reentrant
from ministack.core.responses import (
AccountRegionScopedDict,
AccountScopedDict,
- error_response_json,
get_account_id,
get_region,
new_uuid,
@@ -895,8 +895,9 @@ async def _invoke_request_authorizer_lambda(authorizer, event, account_id, regio
if func_data is None or func_config is None:
return None
exec_record = lambda_svc._execution_record_for_config(func_data, func_config)
- return await asyncio.to_thread(
- lambda_svc._execute_function_with_config_scope, exec_record, event
+ return await run_reentrant(
+ lambda_svc._execute_function_with_config_scope, exec_record, event,
+ thread_name="ministack-apigw-authorizer",
)
@@ -1593,7 +1594,8 @@ async def _invoke_lambda_proxy(
# the shared helper so v1/v2 stay consistent and we get 429s on
# ConcurrentInvocationLimitExceeded.
exec_record = lambda_svc._execution_record_for_config(func_data, func_config)
- result = await asyncio.to_thread(lambda_svc._execute_function_with_config_scope, exec_record, event)
+ result = await run_reentrant(lambda_svc._execute_function_with_config_scope, exec_record, event,
+ thread_name="ministack-apigw-invoke")
lambda_response, _ = lambda_svc.lambda_execute_result_to_api_proxy_response(result)
status = lambda_response.get("statusCode", 200)
@@ -2582,7 +2584,7 @@ async def _invoke_ws_lambda(api_id: str, account_id: str, region: str, route: di
)
return worker.invoke(event, message_id)
- result = await asyncio.to_thread(
+ result = await run_reentrant(
lambda_svc._run_with_function_config_scope,
func_config,
_invoke_worker,
diff --git a/ministack/services/apigateway_v1.py b/ministack/services/apigateway_v1.py
index eef75f6f..86f94d18 100644
--- a/ministack/services/apigateway_v1.py
+++ b/ministack/services/apigateway_v1.py
@@ -84,7 +84,6 @@ Custom domains:
data-plane requests by custom domain host today.
"""
-import asyncio
import base64
import datetime
import json
@@ -98,6 +97,7 @@ import urllib.request
import yaml
from ministack.core.arn import ArnParseError, parse_arn
+from ministack.core.concurrency import run_reentrant
from ministack.core.responses import (
AccountRegionScopedDict,
AccountScopedDict,
@@ -719,7 +719,8 @@ async def _call_lambda_raw(function_ref, event, *, account_id=None, region=None)
return None, f"Lambda function '{label}' not found"
exec_record = lambda_svc._execution_record_for_config(func_data, func_config)
- result = await asyncio.to_thread(lambda_svc._execute_function_with_config_scope, exec_record, event)
+ result = await run_reentrant(lambda_svc._execute_function_with_config_scope, exec_record, event,
+ thread_name="ministack-apigw-invoke")
return result, None
@@ -1395,8 +1396,9 @@ async def _invoke_authorizer_lambda(authorizer, event, account_id, region):
if func_data is None or func_config is None:
return None
exec_record = lambda_svc._execution_record_for_config(func_data, func_config)
- return await asyncio.to_thread(
- lambda_svc._execute_function_with_config_scope, exec_record, event
+ return await run_reentrant(
+ lambda_svc._execute_function_with_config_scope, exec_record, event,
+ thread_name="ministack-apigw-authorizer",
)
diff --git a/ministack/services/appsync_events.py b/ministack/services/appsync_events.py
index 0dab9cd6..e6ade3b8 100644
--- a/ministack/services/appsync_events.py
+++ b/ministack/services/appsync_events.py
@@ -26,6 +26,7 @@ import os
import re
import time
+from ministack.core.concurrency import run_reentrant
from ministack.core.persistence import load_state
from ministack.core.responses import (
AccountRegionScopedDict,
@@ -545,7 +546,7 @@ async def _authorize_event_op(
if not token:
return False, f"{operation} rejected: no Authorization token"
namespace = _channel_namespace_for(channel) if channel else None
- ok, _ctx = await asyncio.to_thread(
+ ok, _ctx = await run_reentrant(
_events_authorizer_invoke,
lambda_arn,
api_id,
@@ -1145,7 +1146,7 @@ async def _handle_websocket_in_api_scope(
):
return
else:
- ok, connect_rctx = await asyncio.to_thread(
+ ok, connect_rctx = await run_reentrant(
_events_authorizer_invoke,
lambda_auth_arn,
api_id,
diff --git a/ministack/services/cloudformation/stacks.py b/ministack/services/cloudformation/stacks.py
index 63ee6c6a..7e123fbd 100644
--- a/ministack/services/cloudformation/stacks.py
+++ b/ministack/services/cloudformation/stacks.py
@@ -4,18 +4,15 @@ CloudFormation stacks — async stack lifecycle (deploy, delete, update, diff).
import asyncio
import copy
-import json
import logging
-import time
from contextlib import contextmanager
-from ministack.core.responses import get_account_id, get_region, new_uuid, now_iso, set_request_region
+from ministack.core.concurrency import run_reentrant
+from ministack.core.responses import get_region, new_uuid, now_iso, set_request_region
from .engine import (
_NO_VALUE,
_evaluate_conditions,
- _parse_template,
- _resolve_parameters,
_resolve_refs,
_topological_sort,
)
@@ -162,7 +159,7 @@ async def _deploy_stack_async(stack_name: str, stack_id: str, template: dict,
old_props = prev_resource.get("Properties", {})
old_attrs = prev_resource.get("Attributes", {})
if _is_custom_resource(resource_type):
- physical_id, attrs = await asyncio.to_thread(
+ physical_id, attrs = await run_reentrant(
_update_resource, resource_type, old_pid, old_props,
resolved_props, stack_name, logical_id, old_attrs
)
@@ -173,7 +170,7 @@ async def _deploy_stack_async(stack_name: str, stack_id: str, template: dict,
)
else:
if _is_custom_resource(resource_type):
- physical_id, attrs = await asyncio.to_thread(
+ physical_id, attrs = await run_reentrant(
_provision_resource, resource_type, logical_id, resolved_props, stack_name
)
else:
@@ -213,7 +210,7 @@ async def _deploy_stack_async(stack_name: str, stack_id: str, template: dict,
old_props = old_res.get("Properties", {})
try:
if _is_custom_resource(rtype):
- await asyncio.to_thread(
+ await run_reentrant(
_delete_resource, rtype, pid, old_props,
stack_name, logical_id
)
@@ -247,7 +244,7 @@ async def _deploy_stack_async(stack_name: str, stack_id: str, template: dict,
res_props = res.get("Properties", {})
try:
if _is_custom_resource(rtype):
- await asyncio.to_thread(
+ await run_reentrant(
_delete_resource, rtype, pid, res_props,
stack_name, logical_id
)
@@ -355,7 +352,7 @@ async def _delete_stack_async(stack_name: str, stack_id: str):
"DELETE_IN_PROGRESS", physical_id=pid)
try:
if _is_custom_resource(rtype):
- await asyncio.to_thread(
+ await run_reentrant(
_delete_resource, rtype, pid, res_props,
stack_name, logical_id
)
diff --git a/ministack/services/codebuild.py b/ministack/services/codebuild.py
index 4f835707..712b5044 100644
--- a/ministack/services/codebuild.py
+++ b/ministack/services/codebuild.py
@@ -21,8 +21,10 @@ import re
import threading
import time
+from ministack.core import container_reaper
from ministack.core.arn import ArnParseError, is_arn, parse_arn
-from ministack.core.persistence import PERSIST_STATE, load_state
+from ministack.core.concurrency import run_reentrant
+from ministack.core.persistence import load_state
from ministack.core.responses import (
AccountRegionScopedDict,
error_response_json,
@@ -34,6 +36,11 @@ from ministack.core.responses import (
logger = logging.getLogger("codebuild")
+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a
+# slow or wedged daemon into a minutes-long stall on a request path.
+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))
+
+
REGION = os.environ.get("MINISTACK_REGION", "us-east-1")
# ---------------------------------------------------------------------------
@@ -218,7 +225,7 @@ def _get_docker():
if _docker is None:
try:
import docker
- _docker = docker.from_env()
+ _docker = docker.from_env(timeout=_DOCKER_TIMEOUT)
except Exception:
logger.exception("Docker unavailable; builds stay metadata-only")
_docker = False
@@ -402,7 +409,7 @@ def _execute_build(build_id, project):
"/var/run/docker.sock": {"bind": "/var/run/docker.sock", "mode": "rw"},
env_dir: {"bind": "/LocalBuild/envFile", "mode": "ro"},
},
- labels={"ministack": "codebuild", "ministack.codebuild.build": build_id},
+ labels=container_reaper.own_labels("codebuild", **{"ministack.codebuild.build": build_id}),
)
except Exception:
logger.exception("Failed to start build %s", build_id)
@@ -423,7 +430,7 @@ def _execute_build(build_id, project):
timed_out.set()
logger.error("Build %s exceeded timeoutInMinutes; stopping it", build_id)
try:
- container.remove(force=True)
+ container.remove(force=True, v=True)
except Exception:
logger.exception("Could not stop the timed-out build %s", build_id)
@@ -476,7 +483,7 @@ def _execute_build(build_id, project):
"phases in this format", build_id, AGENT_IMAGE
)
try:
- container.remove(force=True)
+ container.remove(force=True, v=True)
except Exception:
pass
@@ -485,7 +492,7 @@ def _execute_build(build_id, project):
# Request dispatcher
# ---------------------------------------------------------------------------
-async def handle_request(method, path, headers, body, query_params):
+def _handle_request_sync(method, path, headers, body, query_params):
target = headers.get("x-amz-target", "")
action = target.split(".")[-1] if "." in target else ""
@@ -690,7 +697,7 @@ def _stop_build(data):
container = _container_for_build(bid) if EXECUTE_BUILDS else None
if container:
try:
- container.remove(force=True)
+ container.remove(force=True, v=True)
except Exception:
logger.exception("Could not stop the build container for %s", bid)
@@ -732,3 +739,20 @@ def _batch_delete_builds(data):
else:
not_deleted.append(bid)
return json_response({"buildsDeleted": deleted, "buildsNotDeleted": not_deleted})
+
+
+async def handle_request(method, path, headers, body, query_params):
+ """Dispatch off the event loop.
+
+ Request paths here reach the Docker daemon (container create/start/stop/
+ inspect), which blocks for as long as the daemon takes. Measured on ECS: a
+ cached-image container start held the loop for 7.3s, during which the health
+ endpoint — the cheapest request in the process — could not be served.
+
+ Uses run_reentrant, not the shared pool: containers started here are handed
+ an endpoint pointing back at MiniStack, so one calls in while this
+ dispatch is still running. A bounded pool would queue that nested request
+ behind the call waiting on it.
+ """
+ return await run_reentrant(
+ _handle_request_sync, method, path, headers, body, query_params, thread_name="ministack-codebuild-dispatch")
diff --git a/ministack/services/cognito.py b/ministack/services/cognito.py
index 9c2fec13..731a3a9a 100644
--- a/ministack/services/cognito.py
+++ b/ministack/services/cognito.py
@@ -54,7 +54,6 @@ Wire protocol:
Routing is handled in app.py via two separate SERVICE_HANDLERS entries.
"""
-import asyncio
import base64
import copy
import hashlib
@@ -75,7 +74,8 @@ from xml.etree.ElementTree import tostring as xml_tostring
from defusedxml.ElementTree import fromstring as safe_xml_parse
from ministack.core.arn import ArnParseError, parse_arn
-from ministack.core.persistence import PERSIST_STATE, load_state
+from ministack.core.concurrency import run_reentrant
+from ministack.core.persistence import load_state
from ministack.core.responses import (
AccountRegionScopedDict,
AccountScopedDict,
@@ -1919,7 +1919,8 @@ async def _dispatch_idp(action: str, data: dict):
# back into ministack over HTTP) don't deadlock against a blocked event
# loop. The Lambda response path needs the loop free to accept new
# requests while _execute_function is running.
- return await asyncio.to_thread(_run_idp_handler, handler, action, data)
+ return await run_reentrant(_run_idp_handler, handler, action, data,
+ thread_name="ministack-cognito-trigger")
# ---------------------------------------------------------------------------
@@ -1950,7 +1951,8 @@ async def _dispatch_identity(action: str, data: dict):
handler = handlers.get(action)
if not handler:
return error_response_json("InvalidAction", f"Unknown Cognito Identity action: {action}", 400)
- return await asyncio.to_thread(_run_identity_handler, handler, action, data)
+ return await run_reentrant(_run_identity_handler, handler, action, data,
+ thread_name="ministack-cognito-trigger")
# ===========================================================================
diff --git a/ministack/services/dsql.py b/ministack/services/dsql.py
index d89e8735..e8cd7583 100644
--- a/ministack/services/dsql.py
+++ b/ministack/services/dsql.py
@@ -21,7 +21,8 @@ import string
import threading
import time
-from ministack.core import pgproxy
+from ministack.core import container_reaper, pgproxy
+from ministack.core.concurrency import run_offloop
from ministack.core.persistence import load_state
from ministack.core.responses import (
AccountScopedDict,
@@ -245,7 +246,7 @@ def _run_backend_container(identifier):
},
ports={"5432/tcp": None}, # docker-assigned host port
name=name,
- labels={"ministack": "dsql", "cluster_id": identifier},
+ labels=container_reaper.own_labels("dsql", cluster_id=identifier),
)
if ms_network:
container_kwargs["network"] = ms_network
@@ -306,6 +307,22 @@ def _remove_container(identifier):
logger.debug("DSQL: container removal for %s: %s", identifier, e)
+def _serving_loop():
+ """The loop serving HTTP, or None outside a running server.
+
+ ``_main_loop`` is only populated by the ``start_restored_proxies`` lifespan
+ hook, which is skipped when the dsql module was not already loaded at boot —
+ i.e. on every fresh server whose first dsql call is CreateCluster. The app
+ captures the loop unconditionally, so fall back to that.
+ """
+ try:
+ from ministack import app
+
+ return app._MAIN_LOOP
+ except Exception:
+ return None
+
+
async def _start_backend(identifier, cluster):
"""Background task: container + proxy, then flip the cluster ACTIVE.
@@ -457,10 +474,21 @@ def _create_cluster(data):
_tags[arn] = dict(data["tags"])
if has_backend:
+ # This handler runs on a worker thread (dispatch is off the event loop so
+ # Docker work cannot freeze the server), so there is no *running* loop
+ # here to create_task on. Schedule onto the serving loop instead — the
+ # same pattern _start_backend's respawn path already uses. Falling back
+ # to get_running_loop covers direct/in-process callers that do have one.
+ loop = _main_loop or _serving_loop()
try:
- asyncio.get_running_loop().create_task(_start_backend(identifier, cluster))
+ if loop is not None and loop.is_running():
+ asyncio.run_coroutine_threadsafe(
+ _start_backend(identifier, cluster), loop)
+ else:
+ asyncio.get_running_loop().create_task(
+ _start_backend(identifier, cluster))
except RuntimeError:
- # No event loop (direct/module-level call) — metadata-only.
+ # No event loop at all (module-level call) — metadata-only.
cluster["status"] = "ACTIVE"
cluster["_has_backend"] = False
@@ -742,7 +770,7 @@ def _list_tags_for_resource(arn):
# ---------------------------------------------------------------------------
-async def handle_request(method, path, headers, body, query_params):
+def _handle_request_sync(method, path, headers, body, query_params):
try:
data = json.loads(body) if body else {}
if not isinstance(data, dict):
@@ -905,3 +933,18 @@ try:
restore_state(_restored)
except Exception:
logging.getLogger(__name__).exception("Failed to restore persisted state; continuing with fresh store")
+
+
+async def handle_request(method, path, headers, body, query_params):
+ """Dispatch off the event loop.
+
+ Request paths here reach the Docker daemon (container create/start/stop/
+ inspect), which blocks for as long as the daemon takes. Measured on ECS: a
+ cached-image container start held the loop for 7.3s, during which the health
+ endpoint — the cheapest request in the process — could not be served.
+
+ Uses the shared pool: the containers started here (database / cache / search
+ engines) never call back into MiniStack, so this cannot re-enter.
+ """
+ return await run_offloop(
+ _handle_request_sync, method, path, headers, body, query_params)
diff --git a/ministack/services/ecs.py b/ministack/services/ecs.py
index 373a2b78..209691fc 100644
--- a/ministack/services/ecs.py
+++ b/ministack/services/ecs.py
@@ -31,7 +31,9 @@ import secrets
import threading
import time
+from ministack.core import container_reaper
from ministack.core.arn import ArnParseError, parse_arn
+from ministack.core.concurrency import run_reentrant
from ministack.core.persistence import load_state
from ministack.core.responses import (
AccountRegionScopedDict,
@@ -47,6 +49,11 @@ from ministack.services import ecs_metadata, secretsmanager
logger = logging.getLogger("ecs")
+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a
+# slow or wedged daemon into a minutes-long stall on a request path.
+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))
+
+
REGION = os.environ.get("MINISTACK_REGION", "us-east-1")
_MINISTACK_HOST = os.environ.get("MINISTACK_HOST", "localhost")
_GATEWAY_PORT = os.environ.get("GATEWAY_PORT", "4566")
@@ -80,18 +87,29 @@ _ecs_reaper_started = False
_ecs_reaper_lock = threading.Lock()
+def _live_container_ids():
+ """Container ids still owned by a live task.
+
+ Registered with ``core.container_reaper``, which reclaims ECS containers
+ that no task references any more. This replaces a service-local reaper that
+ removed every exited/dead/created ECS container regardless of ownership —
+ correct in practice, but a second implementation of the same rule, and one
+ that could race a task between `created` and `running`.
+ """
+ ids = set()
+ for _key, task in _tasks.all_items():
+ for cid in task.get("_docker_ids") or []:
+ if cid:
+ ids.add(cid)
+ return ids
+
+
+container_reaper.register_live_ids("ecs", _live_container_ids)
+
+
def _reap_exited_containers():
- docker_client = _get_docker()
- if not docker_client:
- return
- # all=True includes exited containers; reset() uses list() (running only),
- # so this complements — and overlaps harmlessly — with the reset path.
- for c in docker_client.containers.list(all=True, filters={"label": "ministack=ecs"}):
- if c.status in ("exited", "dead", "created"):
- try:
- c.remove(v=True, force=True)
- except Exception:
- pass
+ """Kept as a no-op shim: reclamation is owned by core.container_reaper."""
+ return
def _ensure_ecs_reaper_thread():
@@ -236,7 +254,7 @@ def _get_docker():
if _docker is None:
try:
import docker
- _docker = docker.from_env()
+ _docker = docker.from_env(timeout=_DOCKER_TIMEOUT)
except Exception:
pass
return _docker
@@ -265,7 +283,7 @@ def _ts_to_epoch(value):
if not isinstance(value, str):
return value
try:
- from datetime import datetime, timezone
+ from datetime import datetime
return int(datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp())
except (ValueError, TypeError):
return value
@@ -309,13 +327,26 @@ async def handle_request(method, path, headers, body, query_params):
target = headers.get("x-amz-target", "") or headers.get("X-Amz-Target", "")
if target:
action = target.split(".")[-1]
- return _finalize_ecs_response(_dispatch_action(action, data))
+ # Docker work (RunTask/StopTask/service updates) blocks for as long as
+ # the daemon takes: a cached nginx start already holds the loop >7s,
+ # measured, during which no service in the process can be served.
+ #
+ # run_reentrant, not the shared pool: every task container is handed
+ # AWS_ENDPOINT_URL pointing back here (see _task_env), so a starting
+ # container calls into MiniStack while this call is still in flight. A
+ # bounded pool — or a per-service lock — would put that nested request
+ # behind the call waiting on it.
+ return _finalize_ecs_response(
+ await run_reentrant(_dispatch_action, action, data,
+ thread_name="ministack-ecs-dispatch"))
parts = [p for p in path.strip("/").split("/") if p]
if not parts:
return error_response_json("ClientException", "Missing path", 400)
- return _finalize_ecs_response(_dispatch_path(method, parts, data))
+ return _finalize_ecs_response(
+ await run_reentrant(_dispatch_path, method, parts, data,
+ thread_name="ministack-ecs-dispatch"))
def _dispatch_action(action, data):
@@ -1069,7 +1100,7 @@ def _build_run_kwargs(cdef, td, env, port_bindings, ecs_network,
ports=port_bindings or None,
name=f"ministack-ecs-{task_id[:8]}-{cdef['name']}",
labels={
- "ministack": "ecs",
+ **container_reaper.own_labels("ecs"),
"com.amazonaws.ecs.cluster": cluster_arn,
"com.amazonaws.ecs.container-name": cdef["name"],
"com.amazonaws.ecs.task-arn": task_arn,
@@ -2137,13 +2168,13 @@ def reset():
# (echo, wget probes) leave Exited containers behind that otherwise
# only the background reaper would remove.
try:
- for c in docker_client.containers.list(all=True, filters={"label": "ministack=ecs"}):
- try:
- if c.status == "running":
- c.stop(timeout=2)
- c.remove(v=True, force=True)
- except Exception:
- pass
+ from ministack.core import container_reaper
+ # Concurrently: reset holds the global reset lock, and stopping a
+ # run's worth of tasks one at a time is tens of seconds during which
+ # every other request is blocked.
+ container_reaper.drop_containers(
+ docker_client.containers.list(all=True, filters={"label": "ministack=ecs"}),
+ stop_timeout=2, force=True)
except Exception:
pass
_clusters.clear()
diff --git a/ministack/services/eks.py b/ministack/services/eks.py
index f8728256..90583428 100644
--- a/ministack/services/eks.py
+++ b/ministack/services/eks.py
@@ -24,21 +24,26 @@ import threading
import time
import urllib.parse
+from ministack.core import container_reaper
from ministack.core.arn import ArnParseError, parse_arn
+from ministack.core.concurrency import run_reentrant
from ministack.core.persistence import load_state
from ministack.core.responses import (
AccountRegionScopedDict,
AccountScopedDict,
apply_image_prefix,
- error_response_json,
get_account_id,
get_region,
- json_response,
new_uuid,
)
logger = logging.getLogger("eks")
+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a
+# slow or wedged daemon into a minutes-long stall on a request path.
+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))
+
+
REGION = os.environ.get("MINISTACK_REGION", "us-east-1")
_MINISTACK_HOST = os.environ.get("MINISTACK_HOST", "localhost")
EKS_K3S_IMAGE = os.environ.get("EKS_K3S_IMAGE", "rancher/k3s:v1.31.4-k3s1")
@@ -297,7 +302,7 @@ def _get_docker():
if not _docker_available:
return None
try:
- return docker_lib.from_env()
+ return docker_lib.from_env(timeout=_DOCKER_TIMEOUT)
except Exception:
return None
@@ -418,7 +423,7 @@ def _k3s_run_kwargs(
devices=["/dev/fuse"],
ports={"6443/tcp": port},
name=f"ministack-eks-{region}-{name}",
- labels={"ministack": "eks", "cluster_name": name, "region": region},
+ labels=container_reaper.own_labels("eks", cluster_name=name, region=region),
environment={"K3S_KUBECONFIG_MODE": "644"},
volumes={"/lib/modules": {"bind": "/lib/modules", "mode": "ro"}},
tmpfs={"/run": "", "/var/run": "", "/tmp": ""},
@@ -1454,7 +1459,7 @@ def _sanitize(cluster):
# Request Router
# ---------------------------------------------------------------------------
-async def handle_request(method, path, headers, body_bytes, query_params):
+def _handle_request_sync(method, path, headers, body_bytes, query_params):
try:
body = json.loads(body_bytes) if body_bytes else {}
except json.JSONDecodeError:
@@ -1628,3 +1633,38 @@ async def handle_request(method, path, headers, body_bytes, query_params):
return _untag_resource(arn, query)
return _error(400, "InvalidRequestException", f"No route for {method} {path}")
+
+
+async def handle_request(method, path, headers, body, query_params):
+ """Dispatch off the event loop.
+
+ Request paths here reach the Docker daemon (container create/start/stop/
+ inspect), which blocks for as long as the daemon takes. Measured on ECS: a
+ cached-image container start held the loop for 7.3s, during which the health
+ endpoint — the cheapest request in the process — could not be served.
+
+ Uses run_reentrant, not the shared pool: containers started here are handed
+ an endpoint pointing back at MiniStack, so one calls in while this
+ dispatch is still running. A bounded pool would queue that nested request
+ behind the call waiting on it.
+ """
+ return await run_reentrant(
+ _handle_request_sync, method, path, headers, body, query_params, thread_name="ministack-eks-dispatch")
+
+
+def _live_container_ids():
+ """Container ids still owned by a live EKS cluster.
+
+ EKS records its k3s container under ``_docker_id`` (not the
+ ``_docker_container_id`` the other services use), which is why this was
+ missed first time round and its exited containers were never reclaimed.
+ """
+ ids = set()
+ for _key, cluster in _clusters.all_items():
+ cid = cluster.get("_docker_id")
+ if cid:
+ ids.add(cid)
+ return ids
+
+
+container_reaper.register_live_ids("eks", _live_container_ids)
diff --git a/ministack/services/elasticache.py b/ministack/services/elasticache.py
index 5909700b..d025936a 100644
--- a/ministack/services/elasticache.py
+++ b/ministack/services/elasticache.py
@@ -20,13 +20,16 @@ When Docker is available, CreateCacheCluster spins up a real Redis/Valkey/Memcac
container. Otherwise returns localhost:6379 (assumes Redis sidecar in docker-compose).
"""
+import contextvars
import copy
import logging
import os
import time
from urllib.parse import parse_qs
+from ministack.core import container_reaper
from ministack.core.arn import ArnParseError, parse_arn
+from ministack.core.concurrency import resource_lock, run_offloop, spawn_background
from ministack.core.persistence import load_state
from ministack.core.responses import (
AccountRegionScopedDict,
@@ -257,7 +260,7 @@ def _ensure_live_containers_locked():
name=f"ministack-elasticache-{account_id}-{region}-{name}",
engine=engine, engine_version=version,
labels={
- "ministack": "elasticache",
+ **container_reaper.own_labels("elasticache"),
"cluster_id": name,
"account_id": account_id,
"region": region,
@@ -289,7 +292,7 @@ def _ensure_live_containers_locked():
name=f"ministack-elasticache-rg-{account_id}-{region}-{rg_id}-{ng_id}",
engine=engine, engine_version=engine_version,
labels={
- "ministack": "elasticache", "rg_id": rg_id,
+ **container_reaper.own_labels("elasticache"), "rg_id": rg_id,
"node_group": ng_id, "account_id": account_id,
"region": region,
},
@@ -399,12 +402,27 @@ def _validate_modify_replication_group_request(p):
return None
+# Actions whose implementation talks to the Docker daemon (container run/get/
+# stop). Kept explicit so a new action is off-loop by decision, not by accident.
+_DOCKER_BACKED_ACTIONS = {
+ "CreateCacheCluster", "DeleteCacheCluster", "ModifyCacheCluster",
+ "RebootCacheCluster", "CreateReplicationGroup", "DeleteReplicationGroup",
+ "ModifyReplicationGroup", "IncreaseReplicaCount", "DecreaseReplicaCount",
+ "DescribeCacheClusters", "DescribeReplicationGroups",
+}
+
+
+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a
+# slow or wedged daemon into a minutes-long stall on a request path.
+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))
+
+
def _get_docker():
global _docker
if _docker is None:
try:
import docker
- _docker = docker.from_env()
+ _docker = docker.from_env(timeout=_DOCKER_TIMEOUT)
except Exception:
pass
return _docker
@@ -429,6 +447,20 @@ def _engine_image_and_port(engine, engine_version):
return apply_image_prefix(f"memcached:{engine_version}-alpine"), 11211
+def _image_is_local(docker_client, image: str) -> bool:
+ """True when the image is already pulled.
+
+ ``containers.run`` auto-pulls on ImageNotFound, so calling it with a cold
+ cache blocks the request for as long as the pull takes. AWS returns
+ ``creating`` in milliseconds and provisions asynchronously.
+ """
+ try:
+ docker_client.images.get(image)
+ return True
+ except Exception:
+ return False
+
+
def _spawn_redis_container(name, engine, engine_version, labels):
"""Start a redis/valkey/memcached container.
@@ -519,7 +551,7 @@ def _spawn_redis_cluster_node(name, engine, engine_version, labels):
if not container_ip:
try:
container.stop(timeout=2)
- container.remove()
+ container.remove(v=True)
except Exception:
pass
logger.warning("ElastiCache: cluster node %s has no IP on network %s", name, DOCKER_NETWORK)
@@ -591,7 +623,7 @@ def _teardown_containers(docker_client, container_ids):
try:
c = docker_client.containers.get(cid)
c.stop(timeout=2)
- c.remove()
+ c.remove(v=True)
except Exception as e:
logger.warning("ElastiCache: cleanup failed for %s: %s", cid, e)
@@ -621,7 +653,7 @@ def _build_real_cluster_rg(rg_id, engine, engine_version, num_node_groups, repli
container_ids = []
common_labels = {
- "ministack": "elasticache",
+ **container_reaper.own_labels("elasticache"),
"rg_id": rg_id,
"account_id": account_id,
"region": region,
@@ -866,6 +898,11 @@ async def handle_request(method, path, headers, body, query_params):
handler = handlers.get(action)
if not handler:
return _error("InvalidAction", f"Unknown ElastiCache action: {action}", 400)
+ # Docker-backed actions block on the daemon — a cold image pull is minutes —
+ # and would hold the event loop for every other service. Shared pool, not a
+ # dedicated thread: a Redis/Valkey container never calls back into MiniStack.
+ if action in _DOCKER_BACKED_ACTIONS:
+ return await run_offloop(handler, params)
return handler(params)
@@ -884,17 +921,27 @@ def _create_cache_cluster(p):
arn = _arn_cluster(cluster_id)
account_id = get_account_id()
region = get_region()
- endpoint_host, endpoint_port, docker_container_id = _spawn_redis_container(
+ _spawn_kwargs = dict(
name=f"ministack-elasticache-{account_id}-{region}-{cluster_id}",
engine=engine,
engine_version=engine_version,
labels={
- "ministack": "elasticache",
+ **container_reaper.own_labels("elasticache"),
"cluster_id": cluster_id,
"account_id": account_id,
"region": region,
},
)
+ _image, _port = _engine_image_and_port(engine, engine_version)
+ _dc = _get_docker()
+ _deferred = bool(_dc) and not _image_is_local(_dc, _image)
+ if _deferred:
+ # Cold image cache: pulling inside CreateCacheCluster would block the
+ # API for as long as the pull takes. AWS returns `creating` at once, so
+ # spawn in the background and flip to `available` when it lands.
+ endpoint_host, endpoint_port, docker_container_id = "localhost", _port, None
+ else:
+ endpoint_host, endpoint_port, docker_container_id = _spawn_redis_container(**_spawn_kwargs)
subnet_group = _p(p, "CacheSubnetGroupName") or "default"
param_group_name = _p(p, "CacheParameterGroupName") or _default_param_group_for_engine(engine, engine_version)
@@ -902,7 +949,7 @@ def _create_cache_cluster(p):
_clusters[cluster_id] = {
"CacheClusterId": cluster_id,
"CacheClusterArn": arn,
- "CacheClusterStatus": "available",
+ "CacheClusterStatus": "creating" if _deferred else "available",
"Engine": engine,
"EngineVersion": engine_version,
"CacheNodeType": node_type,
@@ -935,29 +982,103 @@ def _create_cache_cluster(p):
"_endpoint": {"Address": endpoint_host, "Port": endpoint_port},
}
+ if _deferred:
+ # Pull + start off the request path; the cluster reports `creating`
+ # until this lands, exactly as AWS does.
+ created_record = _clusters[cluster_id]
+
+ def _finish_cluster_start():
+ try:
+ host, port, cid = _spawn_redis_container(**_spawn_kwargs)
+ except Exception:
+ with resource_lock("elasticache", cluster_id):
+ rec = _clusters.get(cluster_id)
+ if rec is created_record:
+ rec["CacheClusterStatus"] = "create-failed"
+ return
+ # Publish under the lock so a DeleteCacheCluster cannot land between
+ # the "is it still there" check and the write. Identity, not
+ # truthiness: a same-named cluster created after a delete is a
+ # different resource and must not adopt this container.
+ with resource_lock("elasticache", cluster_id):
+ rec = _clusters.get(cluster_id)
+ orphaned = rec is not created_record or rec.get("_deleting")
+ if not orphaned:
+ rec["_docker_container_id"] = cid
+ _publish_endpoint(rec, host, port)
+ rec["CacheClusterStatus"] = "available"
+ if orphaned:
+ # Deleted while the image was pulling. Nothing references this
+ # container and the periodic reaper only touches created / dead
+ # / exited, never a *running* one — so it would run forever.
+ # Torn down outside the lock: Docker calls never hold it.
+ _teardown_containers(_get_docker(), [cid])
+
+ spawn_background(contextvars.copy_context().run, _finish_cluster_start,
+ thread_name=f"ministack-elasticache-start-{cluster_id}")
+
_tags[arn] = _extract_tags(p)
_record_event(cluster_id, "cache-cluster", "Cache cluster created")
return _xml_cluster_response("CreateCacheClusterResponse", "CreateCacheClusterResult", _clusters[cluster_id])
+def _publish_endpoint(rec: dict, host: str, port: int) -> None:
+ """Write the cluster's endpoint everywhere a reader looks for it.
+
+ The deferred-start bug was two writers disagreeing about the key: create
+ recorded ``_endpoint`` and ``CacheNodes[].Endpoint`` (with the *engine
+ default* port, since no container existed yet), while the finisher updated
+ only ``ConfigurationEndpoint``. Every Describe path reads the first two, so a
+ cold-image create reported ``localhost:6379`` for the rest of its life while
+ the container was on the allocated host port. One writer, so they cannot
+ drift apart again.
+ """
+ endpoint = {"Address": host, "Port": port}
+ rec["_endpoint"] = dict(endpoint)
+ for node in rec.get("CacheNodes") or []:
+ node["Endpoint"] = dict(endpoint)
+ if rec.get("ConfigurationEndpoint") is not None or "ConfigurationEndpoint" in rec:
+ rec["ConfigurationEndpoint"] = dict(endpoint)
+
+
def _delete_cache_cluster(p):
cluster_id = _p(p, "CacheClusterId")
- cluster = _clusters.get(cluster_id)
- if not cluster:
- return _error("CacheClusterNotFound", f"Cluster {cluster_id} not found", 404)
+ # Claim the record under the lock and leave a tombstone, so a deferred
+ # start still in flight sees the delete and reclaims what it started
+ # instead of publishing a container nobody references.
+ with resource_lock("elasticache", cluster_id):
+ cluster = _clusters.get(cluster_id)
+ if not cluster:
+ return _error("CacheClusterNotFound", f"Cluster {cluster_id} not found", 404)
+ cluster["_deleting"] = True
+ cluster["CacheClusterStatus"] = "deleting"
+ container_id = cluster.get("_docker_container_id")
+ del _clusters[cluster_id]
docker_client = _get_docker()
- if docker_client and cluster.get("_docker_container_id"):
- try:
- container = docker_client.containers.get(cluster["_docker_container_id"])
- container.stop(timeout=5)
- container.remove()
- except Exception as e:
- logger.warning("ElastiCache: failed to remove container for %s: %s", cluster_id, e)
-
- cluster["CacheClusterStatus"] = "deleting"
- del _clusters[cluster_id]
+ if docker_client:
+ # By id *or* by the deterministic name. The id alone is not enough: a
+ # delete can arrive before the deferred start has published one, and
+ # then nothing would ever reclaim the container. Two independent
+ # handles, so a leak needs both to miss.
+ for locate in (
+ (lambda: docker_client.containers.get(container_id)) if container_id else None,
+ lambda: docker_client.containers.get(
+ f"ministack-elasticache-{get_account_id()}-{get_region()}-{cluster_id}"),
+ ):
+ if locate is None:
+ continue
+ try:
+ container = locate()
+ except Exception:
+ continue
+ try:
+ container.stop(timeout=5)
+ container.remove(v=True)
+ except Exception as e:
+ logger.warning("ElastiCache: failed to remove container for %s: %s", cluster_id, e)
+ break
_tags.pop(cluster.get("CacheClusterArn", ""), None)
_record_event(cluster_id, "cache-cluster", "Cache cluster deleted")
return _xml_cluster_response("DeleteCacheClusterResponse", "DeleteCacheClusterResult", cluster)
@@ -1108,7 +1229,7 @@ def _create_replication_group(p):
engine=engine,
engine_version=engine_version,
labels={
- "ministack": "elasticache",
+ **container_reaper.own_labels("elasticache"),
"rg_id": rg_id,
"node_group": ng_id,
"account_id": account_id,
@@ -1258,7 +1379,7 @@ def _delete_replication_group(p):
try:
container = docker_client.containers.get(cid)
container.stop(timeout=5)
- container.remove()
+ container.remove(v=True)
except Exception as e:
logger.warning("ElastiCache: failed to remove RG container %s for %s: %s", cid, rg_id, e)
@@ -2416,3 +2537,20 @@ def reset():
_pending_rg_respawn.clear()
_port_counter[0] = BASE_PORT
default_state()
+
+
+def _live_container_ids():
+ ids = set()
+ for _key, rec in _clusters.all_items():
+ cid = rec.get("_docker_container_id")
+ if cid:
+ ids.add(cid)
+ for cid2 in rec.get("_docker_container_ids") or []:
+ ids.add(cid2)
+ for _key, rg in _replication_groups.all_items():
+ for cid2 in rg.get("_docker_container_ids") or []:
+ ids.add(cid2)
+ return ids
+
+
+container_reaper.register_live_ids("elasticache", _live_container_ids)
diff --git a/ministack/services/firehose.py b/ministack/services/firehose.py
index 19e130fb..00299b05 100644
--- a/ministack/services/firehose.py
+++ b/ministack/services/firehose.py
@@ -25,7 +25,8 @@ import threading
import time
from ministack.core.arn import ArnParseError, parse_arn
-from ministack.core.persistence import PERSIST_STATE, load_state
+from ministack.core.concurrency import spawn_background
+from ministack.core.persistence import load_state
from ministack.core.responses import (
AccountRegionScopedDict,
error_response_json,
@@ -521,7 +522,7 @@ def _deliver_to_iceberg(stream: dict, dest: dict, records: list):
if not groups:
return
- def _run():
+ def _deliver_iceberg_groups():
for (wh, db, table, keys), group in groups.items():
try:
_iceberg_write_group(wh, db, table, list(keys), group)
@@ -529,11 +530,14 @@ def _deliver_to_iceberg(stream: dict, dest: dict, records: list):
logger.warning("Firehose %s: Iceberg delivery to %s.%s failed: %s",
name, db, table, exc)
+ # Inline if no thread can be started: losing delivery records silently is
+ # worse than blocking this caller. (Before this ran on `spawn_background`,
+ # the fallback existed for a different reason — `get_running_loop()` raising
+ # off the event loop — which can no longer happen.)
try:
- loop = asyncio.get_running_loop()
- loop.run_in_executor(None, _run)
+ spawn_background(_deliver_iceberg_groups, thread_name="ministack-firehose-iceberg")
except RuntimeError:
- _run()
+ _deliver_iceberg_groups()
def _record_id() -> str:
diff --git a/ministack/services/glue.py b/ministack/services/glue.py
index 111b19a3..a62402f7 100644
--- a/ministack/services/glue.py
+++ b/ministack/services/glue.py
@@ -24,8 +24,10 @@ import threading
import time
from urllib.parse import unquote
+from ministack.core import container_reaper
from ministack.core.arn import ArnParseError, parse_arn
-from ministack.core.persistence import PERSIST_STATE, load_state
+from ministack.core.concurrency import run_reentrant
+from ministack.core.persistence import load_state
from ministack.core.responses import (
AccountRegionScopedDict,
AccountScopedDict,
@@ -38,6 +40,11 @@ from ministack.core.responses import (
logger = logging.getLogger("glue")
+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a
+# slow or wedged daemon into a minutes-long stall on a request path.
+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))
+
+
REGION = os.environ.get("MINISTACK_REGION", "us-east-1")
CRAWLER_RUN_SECONDS = int(os.environ.get("GLUE_CRAWLER_RUN_SECONDS", "5"))
S3_DATA_DIR = os.environ.get("S3_DATA_DIR", "/tmp/ministack-data/s3")
@@ -60,7 +67,7 @@ def _get_docker():
if _docker is None:
try:
import docker
- _docker = docker.from_env()
+ _docker = docker.from_env(timeout=_DOCKER_TIMEOUT)
except Exception:
pass
return _docker
@@ -259,7 +266,7 @@ def _invalid_resource_arn(arn):
return error_response_json("InvalidInputException", f"Invalid Glue resource ARN: {arn}", 400)
-async def handle_request(method, path, headers, body, query_params):
+def _handle_request_sync(method, path, headers, body, query_params):
# Glue's Iceberg REST catalog data plane. On real AWS this is
# `glue.<region>.amazonaws.com/iceberg` — same service, same SigV4
# signing name (`glue`), different protocol (Iceberg REST OpenAPI
@@ -1448,7 +1455,7 @@ def _execute_spark_docker(run, job, job_name, args, script_path, docker_client):
# Remove stale container with same name
try:
existing = docker_client.containers.get(container_name)
- existing.remove(force=True)
+ existing.remove(force=True, v=True)
except Exception:
pass
@@ -1531,7 +1538,7 @@ def _execute_spark_docker(run, job, job_name, args, script_path, docker_client):
"command": cmd,
"environment": container_env,
"detach": True,
- "labels": {"ministack": "glue", "job_name": job_name},
+ "labels": container_reaper.own_labels("glue", job_name=job_name),
}
if ms_network:
@@ -1561,7 +1568,7 @@ def _execute_spark_docker(run, job, job_name, args, script_path, docker_client):
run["JobRunState"] = "FAILED"
run["ErrorMessage"] = f"Docker container start failed: {e}"[:2000]
try:
- container.remove(force=True)
+ container.remove(force=True, v=True)
except Exception:
pass
return
@@ -1585,7 +1592,7 @@ def _execute_spark_docker(run, job, job_name, args, script_path, docker_client):
logger.warning("Glue: Spark container for %s error: %s", job_name, e)
finally:
try:
- container.remove(force=True)
+ container.remove(force=True, v=True)
except Exception:
pass
@@ -2094,3 +2101,20 @@ def reset():
_user_defined_functions.clear()
_table_column_statistics.clear()
_partition_column_statistics.clear()
+
+
+async def handle_request(method, path, headers, body, query_params):
+ """Dispatch off the event loop.
+
+ Request paths here reach the Docker daemon (container create/start/stop/
+ inspect), which blocks for as long as the daemon takes. Measured on ECS: a
+ cached-image container start held the loop for 7.3s, during which the health
+ endpoint — the cheapest request in the process — could not be served.
+
+ Uses run_reentrant, not the shared pool: containers started here are handed
+ an endpoint pointing back at MiniStack, so one calls in while this
+ dispatch is still running. A bounded pool would queue that nested request
+ behind the call waiting on it.
+ """
+ return await run_reentrant(
+ _handle_request_sync, method, path, headers, body, query_params, thread_name="ministack-glue-dispatch")
diff --git a/ministack/services/lambda_svc.py b/ministack/services/lambda_svc.py
index 9f0d95f7..56ec7e41 100644
--- a/ministack/services/lambda_svc.py
+++ b/ministack/services/lambda_svc.py
@@ -24,7 +24,6 @@ with the event piped through stdin (safe from injection).
SQS event source mappings poll the queue in a background thread.
"""
-import asyncio
import base64
import contextvars
import copy
@@ -48,16 +47,20 @@ from datetime import datetime, timezone
from typing import Any
from urllib.parse import quote, unquote
+from ministack.core import container_reaper
from ministack.core.arn import ArnParseError, parse_arn
+from ministack.core.concurrency import run_reentrant
from ministack.core.lambda_runtime import (
INVOKE_DEPTH_BOOTSTRAP,
INVOKE_DEPTH_ENV,
INVOKE_DEPTH_EVENT_KEY,
INVOKE_DEPTH_HEADER,
- get_or_create_worker,
+ acquire_worker,
invalidate_worker,
+ reap_idle_workers,
+ release_worker,
)
-from ministack.core.persistence import PERSIST_STATE, STATE_DIR, load_state
+from ministack.core.persistence import STATE_DIR, load_state
from ministack.core.responses import (
_12_DIGIT_RE,
AccountRegionScopedDict,
@@ -74,6 +77,11 @@ from ministack.core.responses import (
logger = logging.getLogger("lambda")
+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a
+# slow or wedged daemon into a minutes-long stall on a request path.
+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))
+
+
_MINISTACK_HOST = os.environ.get("MINISTACK_HOST", "localhost")
@@ -301,7 +309,7 @@ def _get_docker_client():
if not _docker_available:
return None
try:
- _cached_docker_client = docker_lib.from_env()
+ _cached_docker_client = docker_lib.from_env(timeout=_DOCKER_TIMEOUT)
return _cached_docker_client
except Exception:
return None
@@ -2583,7 +2591,8 @@ async def _invoke(name: str, event: dict, headers: dict, path_qualifier: str | N
# RequestResponse — execute in worker thread so nested SDK calls
# from the Lambda process can still reach this ASGI server.
_t_start = time.time()
- result = await asyncio.to_thread(_execute_function_with_config_scope, exec_record, event)
+ result = await run_reentrant(_execute_function_with_config_scope, exec_record, event,
+ thread_name="ministack-lambda-invoke")
_duration_ms = (time.time() - _t_start) * 1000.0
_emit_lambda_metrics(
name,
@@ -2763,10 +2772,73 @@ _KEEPALIVE_KILL_ON_RELEASE = os.environ.get("LAMBDA_KEEPALIVE_MS", "").strip() =
# Per-function concurrency: only applied when ReservedConcurrentExecutions is
# explicitly set on the function. Otherwise the function can consume the
# full account pool, matching AWS.
-# Account-level concurrency cap: AWS default is 1000. We default to unbounded
-# locally (a laptop can't actually run 1000 Lambda containers); users who
-# want AWS-exact throttling behaviour can set LAMBDA_ACCOUNT_CONCURRENCY.
-_ACCOUNT_CONCURRENCY_CAP = int(os.environ.get("LAMBDA_ACCOUNT_CONCURRENCY", "0")) # 0 = unbounded
+# Per-invocation memory cost of an in-flight execution, measured: a chain of
+# depth 40 took the container to 871 MiB of cgroup memory, depth 20 to 464 MiB.
+_INFLIGHT_MIB = 21
+
+# Fraction of the memory budget concurrency may claim, leaving room for the
+# server itself, page cache, and whatever the handlers allocate.
+_INFLIGHT_BUDGET = 0.5
+
+# AWS's own default account concurrency. Never exceed it: a MiniStack that
+# throttled later than AWS would let code through that AWS rejects.
+_AWS_DEFAULT_ACCOUNT_CONCURRENCY = 1000
+
+
+def _memory_budget_bytes() -> int:
+ """Total memory this process may plan against — cgroup limit if capped."""
+ for path in ("/sys/fs/cgroup/memory.max", # cgroup v2
+ "/sys/fs/cgroup/memory/memory.limit_in_bytes"): # cgroup v1
+ try:
+ with open(path) as fh:
+ raw = fh.read().strip()
+ if raw and raw != "max":
+ value = int(raw)
+ # An uncapped cgroup reports a sentinel near 2**63; ignore it.
+ if 0 < value < (1 << 62):
+ return value
+ except Exception:
+ pass
+ try:
+ return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
+ except Exception:
+ return 2 * 1024 ** 3 # unknowable: assume a small box and stay safe
+
+
+def _default_account_concurrency() -> int:
+ """How many executions may be in flight before we throttle.
+
+ AWS defaults this to 1000 and throttles past it with
+ ``TooManyRequestsException``. MiniStack cannot run 1000 Lambda
+ subprocesses, and the previous default here was **unbounded** — the
+ reasoning being that a laptop can't do 1000 anyway. That is backwards: with
+ no bound, one self-recursing handler takes the process down. It is reachable
+ because ``MAX_INVOKE_DEPTH`` is enforced from a header that only botocore's
+ ``before-call`` hook sets, so a handler whose nested Invoke goes over raw
+ urllib or requests never increments the depth and never trips the limit.
+ Measured: such a handler OOM-killed a 2 GiB container in 8 seconds, 93
+ processes deep, where the depth check never fired once.
+
+ Depth therefore cannot be the safety bound — any transport that bypasses
+ botocore evades it. Concurrency has to be, because every executor passes
+ through ``_acquire_execution_slot`` regardless of how the caller got here.
+
+ Scales to the box so a big machine is not throttled at a small machine's
+ limit, and never exceeds AWS's own 1000. ``LAMBDA_ACCOUNT_CONCURRENCY``
+ still overrides, including ``0`` for genuinely unbounded.
+ """
+ raw = os.environ.get("LAMBDA_ACCOUNT_CONCURRENCY")
+ if raw is not None and raw.strip() != "":
+ try:
+ return int(raw) # explicit wins, 0 still means unbounded
+ except ValueError:
+ logger.warning("LAMBDA_ACCOUNT_CONCURRENCY=%r is not an integer; using the default",
+ raw)
+ budget_mib = (_memory_budget_bytes() * _INFLIGHT_BUDGET) / (1024 ** 2)
+ return max(16, min(_AWS_DEFAULT_ACCOUNT_CONCURRENCY, int(budget_mib // _INFLIGHT_MIB)))
+
+
+_ACCOUNT_CONCURRENCY_CAP = _default_account_concurrency() # 0 = unbounded
_reaper_started = False
_reaper_lock = threading.Lock()
@@ -2795,7 +2867,7 @@ def _kill_pool_entry(entry: dict) -> None:
except Exception:
pass
try:
- container.remove(force=True)
+ container.remove(force=True, v=True)
except Exception:
pass
tmpdir = entry.get("tmpdir")
@@ -2947,6 +3019,13 @@ def _ensure_reaper_thread() -> None:
_pool_evict_idle()
except Exception as exc:
logger.debug("Lambda pool reaper iteration error: %s", exc)
+ try:
+ # Local executor: surplus warm *subprocesses* left behind by
+ # a concurrency burst. Without this the per-function pools
+ # only grow.
+ reap_idle_workers()
+ except Exception as exc:
+ logger.debug("Lambda worker reaper iteration error: %s", exc)
threading.Thread(target=_loop, daemon=True, name="ministack-lambda-reaper").start()
_reaper_started = True
@@ -3706,7 +3785,7 @@ def _spawn_lambda_container(config: dict, code_zip: bytes | None):
"ports": {"8080/tcp": None},
"detach": True,
"stdin_open": False,
- "labels": {"ministack": "lambda"},
+ "labels": container_reaper.own_labels("lambda"),
}
if package_type == "Image":
# User image brings its own entrypoint. ImageConfig can override.
@@ -4039,6 +4118,80 @@ def _emit_lambda_logs(func: dict, request_id: str, log_text: str,
logger.debug("CW Logs emit failed for %s: %s", func.get("config", {}).get("FunctionName"), exc)
+# ---------------------------------------------------------------------------
+# Concurrency accounting — executor-agnostic
+# ---------------------------------------------------------------------------
+#
+# AWS model: concurrent invocations of a function each occupy one unit of
+# concurrency. ReservedConcurrentExecutions caps a single function; the account
+# limit caps every function together. Neither queues — at the limit AWS returns
+# ConcurrentInvocationLimitExceeded (429).
+#
+# This lives at _execute_function rather than in any executor's worker pool
+# because MiniStack has six executors (warm subprocess, docker RIE, provided
+# runtime, proxy, one-off subprocess, and the ESM/WebSocket path) and only some
+# of them own a pool. Counting per pool undercounts every other path, so a
+# mixed deployment would silently never hit the cap.
+_inflight: dict[str, int] = {}
+_inflight_total = 0
+_inflight_lock = threading.Lock()
+
+
+def _inflight_key(config: dict) -> str:
+ try:
+ account, region = _account_region_from_function_config(config)
+ except Exception:
+ account, region = get_account_id(), get_region()
+ return f"{account}:{region}:{config.get('FunctionName', '?')}:{config.get('Version', '$LATEST')}"
+
+
+def _acquire_execution_slot(func: dict, config: dict):
+ """Reserve one unit of concurrency, or None when a cap is reached.
+
+ Never blocks. Waiting for a unit held by your own caller is precisely the
+ deadlock this module exists to avoid: a handler that invokes itself would
+ wait forever on its own slot.
+
+ ``ReservedConcurrentExecutions`` is stored as a bare int on the *function
+ record* under "concurrency" (see _get_function_response), not on the config,
+ so both are consulted rather than assuming either shape.
+ """
+ global _inflight_total
+ reserved = (func or {}).get("concurrency")
+ if isinstance(reserved, dict):
+ reserved = reserved.get("ReservedConcurrentExecutions")
+ if reserved is None:
+ reserved = config.get("ReservedConcurrentExecutions")
+ key = _inflight_key(config)
+ with _inflight_lock:
+ if reserved and _inflight.get(key, 0) >= int(reserved):
+ return None, "function"
+ if _ACCOUNT_CONCURRENCY_CAP > 0 and _inflight_total >= _ACCOUNT_CONCURRENCY_CAP:
+ return None, "account"
+ _inflight[key] = _inflight.get(key, 0) + 1
+ _inflight_total += 1
+ return key, None
+
+
+def _release_execution_slot(key: str) -> None:
+ global _inflight_total
+ if key is None:
+ return
+ with _inflight_lock:
+ remaining = _inflight.get(key, 0) - 1
+ if remaining > 0:
+ _inflight[key] = remaining
+ else:
+ _inflight.pop(key, None)
+ _inflight_total = max(0, _inflight_total - 1)
+
+
+def executions_in_flight() -> tuple[int, dict]:
+ """(total, per-function) invocations currently executing, any executor."""
+ with _inflight_lock:
+ return _inflight_total, dict(_inflight)
+
+
def _execute_function(func: dict, event: dict) -> dict:
"""Dispatch an invocation to the right executor and emit CloudWatch Logs.
@@ -4054,11 +4207,45 @@ def _execute_function(func: dict, event: dict) -> dict:
"""
config = func.get("config") or func
if _function_config_scope_mismatch(config):
+ # Re-entered below under the right scope; counting here would double-book
+ # the slot for a single invocation.
return _run_with_function_config_scope(config, _execute_function, func, event)
request_id = new_uuid()
started = time.time()
+ # Concurrency is accounted here, the one point every executor passes
+ # through, so warm / docker / provided / proxy / one-off all draw on the
+ # same denominator. Accounting inside an executor's own worker pool would
+ # only ever see that executor's share.
+ slot, limit = _acquire_execution_slot(func, config)
+ if slot is None:
+ fn_name = config.get("FunctionName", "unknown")
+ _emit_lambda_metrics(fn_name, duration_ms=0.0, error=False, throttle=True)
+ # AWS models exactly one throttle shape for Invoke —
+ # TooManyRequestsException with a Reason distinguishing the function
+ # limit from the account limit. Anything else is a code string AWS does
+ # not use, which boto3 surfaces as a bare ClientError that
+ # `except lambda.exceptions.TooManyRequestsException` will not catch.
+ if limit == "function":
+ return _throttle_response(
+ reason_code="ReservedFunctionConcurrentInvocationLimitExceeded",
+ msg=f"Rate Exceeded: function {fn_name} at ReservedConcurrentExecutions",
+ )
+ return _throttle_response(
+ reason_code="ConcurrentInvocationLimitExceeded",
+ msg=f"Rate Exceeded: account concurrency cap {_ACCOUNT_CONCURRENCY_CAP} reached",
+ )
+ try:
+ return _execute_function_dispatch(func, config, event, request_id, started)
+ finally:
+ _release_execution_slot(slot)
+
+
+def _execute_function_dispatch(func: dict, config: dict, event: dict,
+ request_id: str, started: float) -> dict:
+ """Executor selection for a single accounted invocation."""
+
# Proxy mode wins over every other executor: the function is bound to a
# user-managed container, ministack only forwards the event.
proxy_url = _proxy_url_for(config)
@@ -4230,8 +4417,21 @@ def _execute_function_warm(func: dict, event: dict) -> dict:
func_name = config.get("FunctionName", "unknown")
qualifier = config.get("Version", "$LATEST")
+ # Concurrent invocations take separate execution environments, as on AWS.
+ # ReservedConcurrentExecutions caps the function; the account cap is checked
+ # across every function. Both report a throttle rather than queueing —
+ # blocking for a free worker would deadlock a handler that invokes itself.
+ # Concurrency (ReservedConcurrentExecutions / account cap) is already
+ # accounted upstream in _execute_function, for every executor. What is left
+ # here is purely the warm-subprocess resource ceiling.
+ _ensure_reaper_thread()
+ worker, reason = acquire_worker(func_name, config, code_zip, qualifier=qualifier)
+ if worker is None and reason == "func_cap":
+ return _throttle_response(
+ reason_code="ReservedFunctionConcurrentInvocationLimitExceeded",
+ msg=f"Rate Exceeded: function {func_name} warm-worker ceiling reached",
+ )
try:
- worker = get_or_create_worker(func_name, config, code_zip, qualifier=qualifier)
# Inject X-Ray trace header into the event so the worker bootstrap
# can set ``_X_AMZN_TRACE_ID`` in os.environ before calling the
# handler. Per-invocation, not bake-time, so it can't live in the
@@ -4265,11 +4465,14 @@ def _execute_function_warm(func: dict, event: dict) -> dict:
except Exception as e:
logger.error("Warm worker execution error for %s: %s", func_name, e)
invalidate_worker(func_name, qualifier=qualifier, account=get_account_id(), region=get_region())
+ worker = None # invalidate_worker already dropped it from the pool
return {
"body": {"errorMessage": str(e), "errorType": type(e).__name__},
"error": True,
"log": "",
}
+ finally:
+ release_worker(worker)
def _execute_function_provided(func: dict, event: dict) -> dict:
@@ -6853,7 +7056,8 @@ async def handle_function_url_request(
event = _build_function_url_event(url_id, account_id, region, method, path, headers, body, query_params)
exec_record = _execution_record_for_config(func_data, func_config)
- result = await asyncio.to_thread(_execute_function_with_config_scope, exec_record, event)
+ result = await run_reentrant(_execute_function_with_config_scope, exec_record, event,
+ thread_name="ministack-lambda-url")
if cfg.get("InvokeMode") == "RESPONSE_STREAM" and not result.get("error") and not result.get("throttle"):
return _function_url_stream_response(result, cors_headers)
diff --git a/ministack/services/mwaa.py b/ministack/services/mwaa.py
index b618ff6f..8f432f0f 100644
--- a/ministack/services/mwaa.py
+++ b/ministack/services/mwaa.py
@@ -21,7 +21,9 @@ import socket
import threading
import time
+from ministack.core import container_reaper
from ministack.core.arn import ArnParseError, parse_arn
+from ministack.core.concurrency import run_offloop
from ministack.core.persistence import load_state
from ministack.core.responses import (
AccountRegionScopedDict,
@@ -134,12 +136,17 @@ def _validate_source_bucket_arn(bucket_arn: str):
return None
+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a
+# slow or wedged daemon into a minutes-long stall on a request path.
+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))
+
+
def _get_docker():
global _docker
if _docker is None:
try:
import docker
- _docker = docker.from_env()
+ _docker = docker.from_env(timeout=_DOCKER_TIMEOUT)
except Exception:
pass
return _docker
@@ -347,7 +354,7 @@ def _start_airflow_container_in_scope(account_id, region, env_name, env):
for stale_name in (container_name, legacy_container_name):
try:
existing = docker_client.containers.get(stale_name)
- existing.remove(force=True)
+ existing.remove(force=True, v=True)
except Exception:
pass
@@ -358,7 +365,7 @@ def _start_airflow_container_in_scope(account_id, region, env_name, env):
environment=container_env,
ports={f"{container_port}/tcp": host_port},
name=container_name,
- labels={"ministack": "mwaa", "region": region, "env_name": env_name},
+ labels=container_reaper.own_labels("mwaa", region=region, env_name=env_name),
)
if ms_network:
@@ -525,7 +532,7 @@ def _delete_environment(method, path, headers, body, query_params):
try:
container = docker_client.containers.get(container_id)
container.stop(timeout=5)
- container.remove(force=True)
+ container.remove(force=True, v=True)
except Exception:
logger.debug("MWAA: container cleanup for %s failed (may be already gone)", name)
@@ -714,13 +721,15 @@ async def handle_request(method, path, headers, body, query_params):
# CRUD on /environments/{Name}
if clean_path.startswith("/environments/"):
if method == "PUT":
- return _create_environment(method, path, headers, body, query_params)
+ # Spawns the Airflow container: a heavyweight image, slow start.
+ # Off-loop; Airflow does not call back into MiniStack.
+ return await run_offloop(_create_environment, method, path, headers, body, query_params)
if method == "GET":
return _get_environment(method, path, headers, body, query_params)
if method == "PATCH":
- return _update_environment(method, path, headers, body, query_params)
+ return await run_offloop(_update_environment, method, path, headers, body, query_params)
if method == "DELETE":
- return _delete_environment(method, path, headers, body, query_params)
+ return await run_offloop(_delete_environment, method, path, headers, body, query_params)
return error_response_json("ValidationException",
f"Unknown MWAA path: {method} {path}", 400)
@@ -735,8 +744,20 @@ def reset():
try:
container = docker_client.containers.get(container_id)
container.stop(timeout=5)
- container.remove(force=True)
+ container.remove(force=True, v=True)
except Exception:
pass
_release_port(env.get("_host_port"))
_environments.clear()
+
+
+def _live_container_ids():
+ ids = set()
+ for _key, env in _environments.all_items():
+ cid = env.get("_docker_container_id")
+ if cid:
+ ids.add(cid)
+ return ids
+
+
+container_reaper.register_live_ids("mwaa", _live_container_ids)
diff --git a/ministack/services/opensearch.py b/ministack/services/opensearch.py
index 9e85706e..af90e718 100644
--- a/ministack/services/opensearch.py
+++ b/ministack/services/opensearch.py
@@ -18,6 +18,7 @@ State is account- and region-scoped via AccountRegionScopedDict, except tags
which stay account-scoped because their ARN keys embed region.
"""
+import contextvars
import copy
import json
import logging
@@ -26,7 +27,9 @@ import re
import threading
import time
+from ministack.core import container_reaper
from ministack.core.arn import ArnParseError, parse_arn
+from ministack.core.concurrency import resource_lock, run_offloop, spawn_background
from ministack.core.persistence import load_state
from ministack.core.responses import (
AccountRegionScopedDict,
@@ -141,7 +144,7 @@ def reset():
):
try:
c.stop(timeout=2)
- c.remove(force=True)
+ c.remove(force=True, v=True)
except Exception:
pass
@@ -307,12 +310,17 @@ def _now() -> int:
# Docker container management
# ---------------------------------------------------------------------------
+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a
+# slow or wedged daemon into a minutes-long stall on a request path.
+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))
+
+
def _get_docker():
global _docker_client
if _docker_client is None:
try:
import docker
- _docker_client = docker.from_env()
+ _docker_client = docker.from_env(timeout=_DOCKER_TIMEOUT)
except Exception:
pass
return _docker_client
@@ -326,6 +334,33 @@ def _dashboards_container_name(domain_name: str) -> str:
return f"ministack-opensearch-dashboards-{get_region()}-{domain_name}"
+def _image_is_local(docker_client, image: str) -> bool:
+ """True when the image is already pulled.
+
+ ``containers.run`` auto-pulls on ImageNotFound, so a cold cache would block
+ CreateDomain for the length of the pull — the OpenSearch image is ~1 GB.
+ AWS reports ``Processing`` and provisions asynchronously.
+ """
+ try:
+ docker_client.images.get(image)
+ return True
+ except Exception:
+ return False
+
+
+def _remove_container_by_id(cid: str) -> None:
+ """Best-effort teardown of a container we started but no longer own."""
+ docker = _get_docker()
+ if not docker or not cid:
+ return
+ try:
+ c = docker.containers.get(cid)
+ c.stop(timeout=2)
+ c.remove(force=True, v=True)
+ except Exception:
+ pass
+
+
def _spawn_dataplane(domain_name: str, engine_version: str):
"""Spawn an OpenSearch container.
@@ -358,6 +393,9 @@ def _spawn_dataplane(domain_name: str, engine_version: str):
image = apply_image_prefix(DEFAULT_IMAGE)
labels = {
+ # Standard label so the boot/shutdown orphan reap finds these; the
+ # com.ministack.* keys below are what the service's own lookups use.
+ **container_reaper.own_labels("opensearch"),
"com.ministack.service": "opensearch",
"com.ministack.domain": domain_name,
"com.ministack.region": get_region(),
@@ -441,7 +479,7 @@ def _teardown_dataplane(rec: dict) -> None:
try:
c = docker.containers.get(cid)
c.stop(timeout=2)
- c.remove(force=True)
+ c.remove(force=True, v=True)
except Exception:
pass
@@ -455,7 +493,7 @@ def _teardown_named_dataplane(domain_name: str) -> None:
try:
c = docker.containers.get(name)
c.stop(timeout=2)
- c.remove(force=True)
+ c.remove(force=True, v=True)
except Exception:
pass
@@ -568,7 +606,15 @@ def _new_domain_record(name, payload):
ebs_opts = _default_ebs_options(payload.get("EBSOptions"))
access_policies = payload.get("AccessPolicies", "")
- host, port, cid, dash_endpoint, dash_cid = _spawn_dataplane(name, engine_version)
+ _dc = _get_docker() if DATAPLANE_ENABLED else None
+ _deferred = bool(_dc) and not _image_is_local(_dc, apply_image_prefix(DEFAULT_IMAGE))
+ if _deferred:
+ # Cold image cache: the OpenSearch image is ~1 GB and pulling it inside
+ # CreateDomain would block the API for minutes. AWS reports
+ # Processing=True and provisions asynchronously; do the same.
+ host, port, cid, dash_endpoint, dash_cid = "localhost", BASE_PORT, None, None, None
+ else:
+ host, port, cid, dash_endpoint, dash_cid = _spawn_dataplane(name, engine_version)
endpoint = f"{host}:{port}"
rec = {
@@ -578,7 +624,7 @@ def _new_domain_record(name, payload):
"Created": True,
"Deleted": False,
"Endpoint": endpoint,
- "Processing": False,
+ "Processing": _deferred,
"UpgradeProcessing": False,
"EngineVersion": engine_version,
"ClusterConfig": cluster_cfg,
@@ -630,6 +676,47 @@ def _new_domain_record(name, payload):
_set_vpc_options(rec, payload.get("VPCOptions"))
if dash_endpoint:
rec["DashboardEndpoint"] = dash_endpoint
+ if _deferred:
+ # Pull + start off the request path. The domain reports Processing=True
+ # until this lands, exactly as AWS does.
+ created_record = rec
+
+ def _finish_domain_start():
+ try:
+ h, pt, cid2, _dep, _dcid = _spawn_dataplane(name, engine_version)
+ except Exception:
+ logger.warning("OpenSearch: deferred dataplane start failed for %s", name)
+ h = pt = cid2 = None
+ # Publish under the lock: a DeleteDomain must not land between the
+ # existence check and the write. Identity, not truthiness — a domain
+ # recreated under the same name is a different resource.
+ with resource_lock("opensearch", name):
+ live = _domains.get(name)
+ orphaned = live is not created_record or live.get("_deleting")
+ if not orphaned:
+ if h:
+ # Both keys. Readers prefer `_Endpoint`, and create left
+ # it at the `localhost:BASE_PORT` placeholder, so
+ # writing only `Endpoint` left Describe reporting an
+ # address nothing listens on — the same defect reported
+ # against ElastiCache.
+ endpoint = f"{h}:{pt}"
+ live["Endpoint"] = endpoint
+ live["_Endpoint"] = endpoint
+ live["_container_id"] = cid2
+ live["_ContainerId"] = cid2
+ live["Processing"] = False
+ if orphaned and cid2:
+ # Nothing references it and the reaper never touches a running
+ # container. Outside the lock: Docker calls never hold it.
+ _remove_container_by_id(cid2)
+
+ # Handed to the caller rather than started here: this function builds
+ # the record, and `_domains[name]` is not assigned until afterwards. A
+ # finisher started now can win that race, find no record, and tear down
+ # the container it just created — the domain then never becomes
+ # available. Started once the record is registered.
+ rec["_deferred_start"] = _finish_domain_start
return rec
@@ -759,6 +846,12 @@ def create_domain_record(payload, compatibility_properties=None):
_tags[rec["ARN"]] = tags
else:
_tags.pop(rec["ARN"], None)
+ # Only now — the finisher looks the record up by name and reclaims what
+ # it started if it is not there, so it must not run before this.
+ deferred_start = rec.pop("_deferred_start", None)
+ if deferred_start is not None:
+ spawn_background(contextvars.copy_context().run, deferred_start,
+ thread_name=f"ministack-opensearch-start-{name}")
return rec
except Exception:
if rec is not None:
@@ -825,7 +918,12 @@ def update_domain_from_cloudformation(name, payload, compatibility_properties):
def delete_domain_record(name, missing_ok=False):
"""Delete all domain-owned state and containers, optionally idempotently."""
- rec = _domains.pop(name, None)
+ # Claim under the lock so a deferred start cannot publish into a record we
+ # are removing. Teardown itself stays outside — it calls Docker.
+ with resource_lock("opensearch", name):
+ rec = _domains.pop(name, None)
+ if rec:
+ rec["_deleting"] = True
if not rec and not missing_ok:
raise OpenSearchServiceError(
404, "ResourceNotFoundException", f"Domain not found: {name}"
@@ -1255,7 +1353,7 @@ async def handle_request(method, path, headers, body_bytes, query_params):
if method == "GET":
return _describe_domain_config(name)
if method == "POST":
- return _update_domain_config(name, payload)
+ return await run_offloop(_update_domain_config, name, payload)
m = _DOMAIN_PROGRESS_RE.match(path)
if method == "GET" and m:
@@ -1263,13 +1361,30 @@ async def handle_request(method, path, headers, body_bytes, query_params):
m = _DOMAIN_RE.match(path)
if method == "POST" and m and m.group("name") is None:
- return _create_domain(payload)
+ # Spawns the OpenSearch (and Dashboards) container: seconds warm,
+ # minutes on a cold pull, and it holds the loop for every other
+ # service meanwhile. Shared pool — an OpenSearch node never calls
+ # back into MiniStack, so this cannot re-enter.
+ return await run_offloop(_create_domain, payload)
if m and m.group("name"):
name = m.group("name")
if method == "GET":
return _describe_domain(name)
if method == "DELETE":
- return _delete_domain(name)
+ return await run_offloop(_delete_domain, name)
return _error(400, "InvalidAction",
f"OpenSearch operation not implemented: {method} {path}")
+
+
+def _live_container_ids():
+ ids = set()
+ for _key, rec in _domains.all_items():
+ for field in ("_container_id", "_dashboards_container_id"):
+ cid = rec.get(field)
+ if cid:
+ ids.add(cid)
+ return ids
+
+
+container_reaper.register_live_ids("opensearch", _live_container_ids)
diff --git a/ministack/services/rds.py b/ministack/services/rds.py
index b735398e..fab8f348 100644
--- a/ministack/services/rds.py
+++ b/ministack/services/rds.py
@@ -46,7 +46,9 @@ import time
from urllib.parse import parse_qs
from xml.sax.saxutils import escape as _esc
+from ministack.core import container_reaper
from ministack.core.arn import ArnParseError, parse_arn
+from ministack.core.concurrency import resource_lock, run_offloop, spawn_background
from ministack.core.persistence import load_state
from ministack.core.responses import (
AccountRegionScopedDict,
@@ -67,6 +69,11 @@ from ministack.services.rds_mysql_compat import (
logger = logging.getLogger("rds")
+# Cap any single Docker daemon call. docker-py defaults to 60s, which turns a
+# slow or wedged daemon into a minutes-long stall on a request path.
+_DOCKER_TIMEOUT = float(os.environ.get("MINISTACK_DOCKER_TIMEOUT", "10"))
+
+
REGION = os.environ.get("MINISTACK_REGION", "us-east-1")
_MINISTACK_HOST = os.environ.get("MINISTACK_HOST", "localhost")
BASE_PORT = int(os.environ.get("RDS_BASE_PORT", "15432"))
@@ -1037,7 +1044,7 @@ def _start_cluster_shared_container(cluster_id, cluster, remove_stale=False):
ports={f"{container_port}/tcp": host_port},
name=container_name,
labels={
- "ministack": "rds",
+ **container_reaper.own_labels("rds"),
"cluster_id": cluster_id,
"account_id": get_account_id(),
"region": get_region(),
@@ -1913,6 +1920,21 @@ def _restart_cluster_shared_container(cluster_id, cluster):
}
+def _image_is_local(docker_client, image: str) -> bool:
+ """True when the image is already pulled.
+
+ ``containers.run`` auto-pulls on ImageNotFound, so calling it with a cold
+ cache blocks the request for as long as the pull takes — minutes for the
+ database and Airflow images. AWS returns ``creating`` in milliseconds and
+ provisions asynchronously, so a missing image must not be pulled inline.
+ """
+ try:
+ docker_client.images.get(image)
+ return True
+ except Exception:
+ return False
+
+
def _start_rds_container_for_instance(db_id, instance):
"""Re-spin (or re-attach to) the Docker container for a restored instance.
@@ -2001,7 +2023,7 @@ def _start_rds_container_for_instance(db_id, instance):
ports={f"{container_port}/tcp": host_port},
name=container_name,
labels={
- "ministack": "rds",
+ **container_reaper.own_labels("rds"),
"db_id": db_id,
"account_id": get_account_id(),
"region": get_region(),
@@ -2027,7 +2049,27 @@ def _start_rds_container_for_instance(db_id, instance):
instance["DBInstanceStatus"] = "failed"
return
- instance["_docker_container_id"] = container.id
+ # Check-and-publish under the lock, so a DeleteDBInstance cannot land in the
+ # gap between them and leave this container referenced by nothing. The check
+ # has to be unconditional — the MySQL readiness path below has its own, but
+ # Postgres would otherwise leave the container running forever.
+ with resource_lock("rds", db_id):
+ current = _instances.get(db_id)
+ orphaned = current is not instance or instance.get("_deleting")
+ if not orphaned:
+ instance["_docker_container_id"] = container.id
+ if orphaned:
+ # Deleted (or replaced) while we were pulling and starting. Reclaim the
+ # container: no record references it, and the periodic reaper never
+ # touches a *running* container, so nothing else ever would. Outside the
+ # lock — Docker calls never hold it.
+ logger.info("RDS: instance %s vanished during start; reclaiming container", db_id)
+ try:
+ container.stop(timeout=2)
+ container.remove(force=True, v=True)
+ except Exception:
+ pass
+ return
internal_host = None
internal_port = None
@@ -2062,6 +2104,15 @@ def _start_rds_container_for_instance(db_id, instance):
_instances.get(db_id) is not instance
or instance.get("_docker_container_id") != container.id
):
+ # The instance was deleted (or replaced) while we were pulling and
+ # starting. Reclaim what we started: nothing else can, because the
+ # periodic reaper never touches a *running* container and no record
+ # references this id any more.
+ try:
+ container.stop(timeout=2)
+ container.remove(force=True, v=True)
+ except Exception:
+ pass
return
instance["DBInstanceStatus"] = "available"
logger.info("RDS: respawned container %s for instance %s",
@@ -2073,7 +2124,7 @@ def _get_docker():
if _docker is None:
try:
import docker
- _docker = docker.from_env()
+ _docker = docker.from_env(timeout=_DOCKER_TIMEOUT)
except Exception:
pass
return _docker
@@ -2964,7 +3015,7 @@ def _flatten_json_request_params(params, data):
params[f"Filters.member.{i}.Values.member.{j}"] = [str(v)]
-async def handle_request(method, path, headers, body, query_params):
+def _handle_request_sync(method, path, headers, body, query_params):
params = dict(query_params)
if method == "POST" and body:
raw = body if isinstance(body, str) else body.decode("utf-8-sig", errors="replace")
@@ -3454,6 +3505,7 @@ def _create_db_instance_impl(p):
readiness_master_pass = master_pass
ms_network = None
+ deferred_container_start = False
docker_client = _get_docker()
reader_launch = None
if (
@@ -3553,7 +3605,16 @@ def _create_db_instance_impl(p):
image, env, container_port, data_path = _docker_image_for_engine(
engine, engine_version, master_user, master_pass, db_name
)
- if image:
+ if image and not _image_is_local(docker_client, image):
+ # Cold image cache: pulling here would block CreateDBInstance for
+ # minutes (measured: 187s for postgres:15-alpine). AWS returns
+ # `creating` at once, so allocate the port, defer the whole start to
+ # the background finaliser below, and let DescribeDBInstances show
+ # `available` when it lands.
+ host_port = _next_port()
+ endpoint_port = host_port
+ deferred_container_start = True
+ elif image:
try:
# Create path: the `instance` dict doesn't exist yet
# (it's built ~70 lines below). Allocate a fresh free port
@@ -3567,7 +3628,7 @@ def _create_db_instance_impl(p):
ports={f"{container_port}/tcp": host_port},
name=_rds_docker_name(db_id),
labels={
- "ministack": "rds",
+ **container_reaper.own_labels("rds"),
"db_id": db_id,
"account_id": get_account_id(),
"region": get_region(),
@@ -3629,6 +3690,10 @@ def _create_db_instance_impl(p):
"DBSubnetGroupArn": f"arn:aws:rds:{get_region()}:{get_account_id()}:subgrp:{subnet_group_name}",
})
instance_status = "creating" if real_container_started else "available"
+ if deferred_container_start:
+ # The container is being pulled and started in the background; AWS
+ # reports `creating` until provisioning completes.
+ instance_status = "creating"
if parent and not parent.get("_shared_container_ready", True):
instance_status = "creating"
if parent and start_result.get("failed"):
@@ -3737,6 +3802,18 @@ def _create_db_instance_impl(p):
_instances[db_id] = instance
_register_instance_in_cluster(instance)
+ if deferred_container_start:
+ # Cold image cache. Pull and start on a background thread — the same
+ # finaliser persistence restore uses — so CreateDBInstance returns now
+ # with status="creating", as AWS does, instead of blocking on the pull.
+ ctx = contextvars.copy_context()
+
+ def _deferred_start():
+ _start_rds_container_for_instance(db_id, instance)
+
+ spawn_background(ctx.run, _deferred_start,
+ thread_name=f"ministack-rds-start-{db_id}")
+
if real_container_started and reader_launch:
# Replicating readers have their own readiness contract (wait for
# the writer, provision replication, wait for the standby), so they
@@ -4002,15 +4079,35 @@ def _delete_db_instance(p):
else:
_stop_cluster_shared_container(shared_cluster_id, cluster)
+ # Tombstone first, under the lock: a deferred start still pulling would
+ # otherwise publish its container id after we read it here, and the record
+ # is gone by the time it lands — leaving a running container nothing
+ # references. With the tombstone that finisher reclaims what it started.
+ with resource_lock("rds", instance_id):
+ instance["_deleting"] = True
+ owned_id = instance.get("_docker_container_id") if _instance_owns_container(instance) else None
+
docker_client = _get_docker()
- if docker_client and _instance_owns_container(instance):
- try:
- c = docker_client.containers.get(instance["_docker_container_id"])
- c.stop(timeout=5)
- c.remove(v=True)
- logger.info("RDS: removed container for %s", instance_id)
- except Exception as e:
- logger.warning("RDS: failed to remove container for %s: %s", instance_id, e)
+ if docker_client:
+ # By id, else by the deterministic name — the id may never have been
+ # published if the delete beat the deferred start.
+ for locate in (
+ (lambda: docker_client.containers.get(owned_id)) if owned_id else None,
+ lambda: docker_client.containers.get(_rds_docker_name(instance_id)),
+ ):
+ if locate is None:
+ continue
+ try:
+ c = locate()
+ except Exception:
+ continue
+ try:
+ c.stop(timeout=5)
+ c.remove(v=True)
+ logger.info("RDS: removed container for %s", instance_id)
+ except Exception as e:
+ logger.warning("RDS: failed to remove container for %s: %s", instance_id, e)
+ break
skip_snapshot = _p(p, "SkipFinalSnapshot") == "true"
final_snap_id = _p(p, "FinalDBSnapshotIdentifier")
@@ -7644,3 +7741,39 @@ except Exception:
logging.getLogger(__name__).exception(
"Failed to restore persisted state; continuing with fresh store"
)
+
+
+async def handle_request(method, path, headers, body, query_params):
+ """Dispatch off the event loop.
+
+ Request paths here reach the Docker daemon (container create/start/stop/
+ inspect), which blocks for as long as the daemon takes. Measured on ECS: a
+ cached-image container start held the loop for 7.3s, during which the health
+ endpoint — the cheapest request in the process — could not be served.
+
+ Uses the shared pool: the containers started here (database / cache / search
+ engines) never call back into MiniStack, so this cannot re-enter.
+ """
+ return await run_offloop(
+ _handle_request_sync, method, path, headers, body, query_params)
+
+
+def _live_container_ids():
+ """Container ids still owned by a live DB instance or cluster.
+
+ A stopped instance still owns its (exited) container — StartDBInstance must
+ be able to restart it — so it is reported here and never reaped.
+ """
+ ids = set()
+ for _key, inst in _instances.all_items():
+ cid = inst.get("_docker_container_id")
+ if cid:
+ ids.add(cid)
+ for _key, cl in _clusters.all_items():
+ cid = cl.get("_shared_container_id")
+ if cid:
+ ids.add(cid)
+ return ids
+
+
+container_reaper.register_live_ids("rds", _live_container_ids)
diff --git a/ministack/services/stepfunctions.py b/ministack/services/stepfunctions.py
index b5b14f7e..9600519e 100644
--- a/ministack/services/stepfunctions.py
+++ b/ministack/services/stepfunctions.py
@@ -39,7 +39,8 @@ from concurrent.futures import wait as futures_wait
from datetime import datetime, timezone
from ministack.core.arn import ArnParseError, parse_arn
-from ministack.core.persistence import PERSIST_STATE, load_state
+from ministack.core.concurrency import run_reentrant
+from ministack.core.persistence import load_state
from ministack.core.responses import (
AccountRegionScopedDict,
AccountScopedDict,
@@ -236,6 +237,12 @@ def _finalize_response(response):
# Entry point
# ---------------------------------------------------------------------------
+# Actions that run a state machine to completion inside the request. They
+# re-enter MiniStack (Lambda invokes, aws-sdk integrations), so they run on a
+# dedicated thread rather than holding the event loop those nested calls need.
+_OFF_LOOP_ACTIONS = {"StartSyncExecution", "TestState"}
+
+
async def handle_request(method, path, headers, body, query_params):
target = headers.get("x-amz-target", "")
action = target.split(".")[-1] if "." in target else ""
@@ -286,6 +293,15 @@ async def handle_request(method, path, headers, body, query_params):
return error_response_json("InvalidAction", f"Unknown action: {action}", 400)
if action == "GetActivityTask":
return _finalize_response(await _get_activity_task(data))
+ if action in _OFF_LOOP_ACTIONS:
+ # These run a state machine to completion inline. Every task state calls
+ # back into MiniStack — Lambda invokes, aws-sdk service integrations —
+ # so this must not hold the event loop: the nested calls need it, and a
+ # service handler that awaits (Docker work runs off-loop now) cannot be
+ # driven from the loop thread at all. Dedicated thread, not the shared
+ # pool, because the work re-enters the server.
+ return _finalize_response(
+ await run_reentrant(handler, data, thread_name="ministack-sfn-sync-exec"))
return _finalize_response(handler(data))
@@ -4386,25 +4402,14 @@ def _dispatch_aws_sdk_json(service_info, service_name, action, input_data):
),
}
- # Service handlers are async def but perform no real I/O, so we can
- # drive the coroutine synchronously — this avoids conflicts with the
- # already-running asyncio event loop.
- coro = handler("POST", "/", headers, body, {})
- try:
- coro.send(None)
- except StopIteration as stop:
- status, resp_headers, resp_body = stop.value
- else:
- # If the coroutine didn't finish in one step it truly needs async;
- # fall back to the event loop (only reachable if a handler awaits).
- coro.close()
- loop = asyncio.new_event_loop()
- try:
- status, resp_headers, resp_body = loop.run_until_complete(
- handler("POST", "/", headers, body, {})
- )
- finally:
- loop.close()
+ # Service handlers are async def and may now genuinely await (services that
+ # drive Docker run that work off the event loop), so they cannot be driven
+ # by hand with ``coro.send(None)``: the first step would execute with no
+ # running loop and raise "no running event loop". Schedule on the loop that
+ # is actually serving and wait here. This runs on a Step Functions execution
+ # thread, never on the loop itself, so the wait cannot deadlock.
+ status, resp_headers, resp_body = app.call_service_handler_sync(
+ handler, "POST", "/", headers, body, {})
decoded = resp_body.decode("utf-8") if isinstance(resp_body, bytes) else resp_body
result = json.loads(decoded) if decoded else {}
@@ -4989,20 +4994,9 @@ def _dispatch_aws_sdk_query(service_info, service_name, action, input_data):
),
}
- coro = handler("POST", "/", headers, body, {})
- try:
- coro.send(None)
- except StopIteration as stop:
- status, resp_headers, resp_body = stop.value
- else:
- coro.close()
- loop = asyncio.new_event_loop()
- try:
- status, resp_headers, resp_body = loop.run_until_complete(
- handler("POST", "/", headers, body, {})
- )
- finally:
- loop.close()
+ # Scheduled on the serving loop — see _dispatch_aws_sdk_json.
+ status, resp_headers, resp_body = app.call_service_handler_sync(
+ handler, "POST", "/", headers, body, {})
decoded = resp_body.decode("utf-8") if isinstance(resp_body, bytes) else resp_body
@@ -5092,20 +5086,9 @@ def _dispatch_aws_sdk_rest_json(service_info, service_name, action, input_data):
),
}
- coro = handler("POST", path, headers, body, {})
- try:
- coro.send(None)
- except StopIteration as stop:
- status, resp_headers, resp_body = stop.value
- else:
- coro.close()
- loop = asyncio.new_event_loop()
- try:
- status, resp_headers, resp_body = loop.run_until_complete(
- handler("POST", path, headers, body, {})
- )
- finally:
- loop.close()
+ # Scheduled on the serving loop — see _dispatch_aws_sdk_json.
+ status, resp_headers, resp_body = app.call_service_handler_sync(
+ handler, "POST", path, headers, body, {})
decoded = resp_body.decode("utf-8") if isinstance(resp_body, bytes) else resp_body
@@ -5226,26 +5209,11 @@ def _dispatch_aws_sdk_lambda_rest(service_info, service_name, action, input_data
),
}
- # Drive the async handler synchronously. SFN state execution runs inside
- # the request's event loop, so spawning a fresh loop here would raise
- # "Cannot run the event loop while another loop is running". The Lambda
- # REST handlers we dispatch to here only touch in-memory dicts and never
- # await, so a single ``coro.send(None)`` completes via ``StopIteration``
- # with the response tuple.
- coro = handler(method, path, headers, body, query_params)
- try:
- coro.send(None)
- except StopIteration as stop:
- status, resp_headers, resp_body = stop.value
- else:
- # Fallback for an async handler that does await — extremely rare on
- # this dispatch surface, but guard so we don't return None silently.
- coro.close()
- raise _ExecutionError(
- "States.Runtime",
- f"aws-sdk:{service_name}:{action} handler awaited unexpectedly; "
- "cannot drive from sync SFN executor",
- )
+ # Scheduled on the serving loop — see _dispatch_aws_sdk_json. This replaces
+ # a guard that failed the execution outright when a handler awaited; that is
+ # now an ordinary, supported case.
+ status, resp_headers, resp_body = app.call_service_handler_sync(
+ handler, method, path, headers, body, query_params)
decoded = resp_body.decode("utf-8") if isinstance(resp_body, bytes) else resp_body
if status >= 400:
@@ -5577,20 +5545,9 @@ def _dispatch_aws_sdk_rest_xml(service_info, service_name, action, input_data):
# passing the raw path with "?..." would treat "?list-type=2" as part of
# the bucket name. Send the query-less path; the dict is the source of
# truth for query routing.
- coro = handler(method, path, headers, body, query_dict)
- try:
- coro.send(None)
- except StopIteration as stop:
- status, resp_headers, resp_body = stop.value
- else:
- coro.close()
- loop = asyncio.new_event_loop()
- try:
- status, resp_headers, resp_body = loop.run_until_complete(
- handler(method, path, headers, body, query_dict)
- )
- finally:
- loop.close()
+ # Scheduled on the serving loop — see _dispatch_aws_sdk_json.
+ status, resp_headers, resp_body = app.call_service_handler_sync(
+ handler, method, path, headers, body, query_dict)
decoded = resp_body.decode("utf-8") if isinstance(resp_body, bytes) else (resp_body or "")
norm_resp_headers = {k.lower(): v for k, v in (resp_headers or {}).items()}
diff --git a/ministack/services/transfer.py b/ministack/services/transfer.py
index 85d16442..4275f195 100644
--- a/ministack/services/transfer.py
+++ b/ministack/services/transfer.py
@@ -38,9 +38,8 @@ import io
import json
import logging
import os
-import re
import time
-from typing import AsyncIterator, Optional
+from typing import Optional
from ministack.core.persistence import load_state
from ministack.core.responses import (
@@ -213,6 +212,16 @@ async def _create_server(data):
_servers[sid] = server
+ # Bind the shared listener here rather than at boot. Nobody can connect to
+ # a server they have not created, so binding on CreateServer is equivalent
+ # — and it keeps `asyncssh` (which drags in cryptography + OpenSSL, ~26 MB
+ # of heap) out of every MiniStack that never touches Transfer Family.
+ # Idempotent, so repeated CreateServer calls cost one dict lookup.
+ try:
+ await sftp_start()
+ except Exception as e:
+ logger.warning("SFTP listener failed to start for %s: %s", sid, e)
+
# If SFTP_PORT_PER_SERVER=1, allocate a dedicated SFTP listener for
# this server (matches AWS's per-server endpoint model). Otherwise
# the shared :2222 listener handles every server, disambiguating by
@@ -1219,6 +1228,19 @@ def _ensure_sftp_runtime() -> None:
# ── SFTP lifecycle — called from app.py lifespan + Create/Delete handlers ──
+def has_servers() -> bool:
+ """True if any Transfer server exists, in any account/region.
+
+ Used at boot to decide whether the SFTP listener has to come up before a
+ CreateServer call would bring it up on its own — i.e. only when persistence
+ restored servers that a client may connect to immediately.
+ """
+ try:
+ return bool(_servers.to_dict())
+ except Exception:
+ return False
+
+
async def sftp_start() -> None:
"""Idempotent: bind the shared SFTP listener on `SFTP_PORT` if enabled."""
if not _sftp_enabled():
diff --git a/tests/conftest.py b/tests/conftest.py
index f5d2756b..5675fe02 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -2,10 +2,17 @@
Pytest fixtures for MiniStack integration tests.
"""
+import concurrent.futures
import contextlib
+import io
import os
import socket
+import statistics
+import threading
+import time
import urllib.request
+import uuid
+import zipfile
from urllib.parse import urlparse
import boto3
@@ -719,3 +726,171 @@ def dsql():
@pytest.fixture(scope="session")
def s3tables():
return make_client("s3tables")
+
+
+class FakeDockerContainer:
+ """Container double for tests that observe lifecycle without a daemon."""
+
+ def __init__(self, daemon, name, labels):
+ self.daemon, self.name, self.labels = daemon, name, dict(labels or {})
+ self.id = f"cid-{name}"
+ self.status = "running"
+ self.attrs = {"Config": {"Labels": self.labels}, "State": {}, "Created": ""}
+
+ def stop(self, timeout=None):
+ self.status = "exited"
+
+ def remove(self, force=False, v=False):
+ self.daemon.removed.append(self.name)
+ self.daemon.containers._by_name.pop(self.name, None)
+
+ def reload(self):
+ pass
+
+
+class _FakeContainers:
+ def __init__(self, daemon):
+ self.daemon = daemon
+ self._by_name = {}
+
+ def run(self, image=None, command=None, name=None, labels=None, **kw):
+ self._by_name[name] = FakeDockerContainer(self.daemon, name, labels)
+ return self._by_name[name]
+
+ def get(self, key):
+ for c in self._by_name.values():
+ if key in (c.name, c.id):
+ return c
+ raise RuntimeError(f"no such container: {key}")
+
+ def list(self, all=False, filters=None):
+ want = (filters or {}).get("label") or []
+ want = [want] if isinstance(want, str) else want
+ out = []
+ for c in self._by_name.values():
+ ok = True
+ for sel in want:
+ if "=" in sel:
+ k, v = sel.split("=", 1)
+ ok = ok and c.labels.get(k) == v
+ else:
+ ok = ok and sel in c.labels
+ if ok:
+ out.append(c)
+ return out
+
+
+class FakeDockerDaemon:
+ """Enough of the Docker API to assert which containers survive.
+
+ ``images.get`` always raises, so services that check whether an image is
+ cached take their deferred/background provisioning path.
+ """
+
+ def __init__(self):
+ self.containers = _FakeContainers(self)
+ self.removed = []
+ self.images = self
+
+ def get(self, image):
+ raise RuntimeError("image not present")
+
+ def live(self):
+ return sorted(self.containers._by_name)
+
+
+def fake_docker():
+ return FakeDockerDaemon()
+
+
+# ---------------------------------------------------------------------------
+# Concurrency probes — shared by the per-service re-entrancy tests
+# ---------------------------------------------------------------------------
+
+# Enough callers to exhaust a misclassified bounded pool without being slow.
+CONCURRENCY_N = 12
+# The nested-invoke probe has to exceed the shared worker pool (default 64) or
+# starvation never shows: below the pool size every caller gets a slot and a
+# broken build passes. Verified — at N=12 the probe passes against 1.4.17,
+# which has the bug.
+CONCURRENCY_N_NESTED = 70
+# Judged on the *median*, never the max. A single slow sample says nothing about
+# the event loop: the probe competes with the burst for client connections and
+# CI runner CPU, so the worst sample is dominated by queueing on our side.
+# Measured spread — a build with the bug blocks the loop for the whole burst and
+# every sample is slow (1.4.17: median 6905ms), while a healthy one absorbs it
+# (median 118ms, with an occasional multi-second outlier under a saturated
+# connection pool). The median separates those by ~60x; the max does not
+# separate them at all.
+MEDIAN_LOOP_STALL_MS = 2000
+
+
+class LoopProbe:
+ """Polls /_ministack/health through a burst to detect a blocked event loop.
+
+ Used by the re-entrancy tests: work that can call back into MiniStack must
+ run on a dedicated thread, and work that blocks must never run on the loop.
+ Both failures show up here as the health endpoint going slow or silent.
+ """
+
+ MIN_SAMPLES = 3 # below this an outlier decides the median; too short to judge
+
+ def __init__(self):
+ self.samples = []
+ self._stop = threading.Event()
+ self._thread = None
+
+ def __enter__(self):
+ def poll():
+ while not self._stop.is_set():
+ t0 = time.perf_counter()
+ try:
+ urllib.request.urlopen(f"{ENDPOINT}/_ministack/health", timeout=15).read()
+ self.samples.append((time.perf_counter() - t0) * 1000)
+ except Exception:
+ self.samples.append(None)
+ time.sleep(0.05)
+
+ self._thread = threading.Thread(target=poll, daemon=True)
+ self._thread.start()
+ return self
+
+ def __exit__(self, *exc):
+ self._stop.set()
+ if self._thread:
+ self._thread.join(timeout=5)
+
+ def assert_responsive(self, what: str):
+ served = [s for s in self.samples if s is not None]
+ assert served, f"{what}: health never responded — the event loop was blocked"
+ if len(served) < self.MIN_SAMPLES:
+ return
+ median = statistics.median(served)
+ assert median < MEDIAN_LOOP_STALL_MS, (
+ f"{what}: event loop median {median:.0f}ms "
+ f"(worst {max(served):.0f}ms, n={len(self.samples)}, "
+ f"unanswered={len(self.samples) - len(served)}) — "
+ f"a blocking call is running on the loop"
+ )
+
+
+def concurrent_burst(fn, items=None, n=CONCURRENCY_N):
+ """Run `fn` concurrently over `items` (or over range(n)); return all results."""
+ work = list(items) if items is not None else list(range(n))
+ with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, len(work))) as ex:
+ return list(ex.map(fn, work))
+
+
+def make_probe_lambda(lam, src: str, timeout: int = 30) -> str:
+ """Deploy a throwaway Python function and return its name."""
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w") as z:
+ z.writestr("index.py", src)
+ name = f"conc-{uuid.uuid4().hex[:10]}"
+ lam.create_function(
+ FunctionName=name, Runtime="python3.11",
+ Role="arn:aws:iam::000000000000:role/lambda-role", Handler="index.handler",
+ Code={"ZipFile": buf.getvalue()}, Timeout=timeout,
+ )
+ return name
diff --git a/tests/test_apigatewayv2.py b/tests/test_apigatewayv2.py
index d190df4b..d3d20464 100644
--- a/tests/test_apigatewayv2.py
+++ b/tests/test_apigatewayv2.py
@@ -6,6 +6,8 @@ import json
import os
import threading
import time
+import urllib.error
+import urllib.request
import uuid as _uuid_mod
import zipfile
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -13,6 +15,13 @@ from urllib.parse import urlparse
import pytest
from botocore.exceptions import ClientError
+from conftest import (
+ CONCURRENCY_N,
+ ENDPOINT,
+ LoopProbe,
+ concurrent_burst,
+ make_probe_lambda,
+)
_endpoint = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566")
@@ -4150,3 +4159,43 @@ def test_apigwv2_authorizer_cache_is_bounded():
finally:
cache.clear()
cache.update(saved)
+
+
+# ---------------------------------------------------------------------------
+# Re-entrancy under concurrency — the rest of the suite issues one request at
+# a time, so a service whose blocking work is misclassified passes serially
+# and only wedges under load. These fire N callers at once and assert both
+# that every caller completes and that the event loop keeps serving.
+# ---------------------------------------------------------------------------
+def test_concurrent_apigw_lambda_proxy_requests(apigw, lam):
+ """N simultaneous HTTP-API requests, each executing a Lambda, must all serve."""
+ fn = make_probe_lambda(lam, (
+ "import time\n"
+ "def handler(event, context):\n"
+ " time.sleep(0.4)\n"
+ " return {'statusCode': 200, 'body': 'ok'}\n"
+ ))
+ api = apigw.create_api(Name=f"conc-{uuid.uuid4().hex[:8]}", ProtocolType="HTTP",
+ Target=f"arn:aws:lambda:us-east-1:000000000000:function:{fn}")
+ api_id = api["ApiId"]
+
+ def call(_):
+ try:
+ r = urllib.request.urlopen(f"{ENDPOINT}/_apigw/{api_id}/", timeout=30)
+ return r.status
+ except urllib.error.HTTPError as e:
+ return e.code
+ except Exception:
+ return None
+
+ with LoopProbe() as probe:
+ codes = concurrent_burst(call)
+ probe.assert_responsive("apigw proxy burst")
+
+ # The route wiring differs by MiniStack version; what this test guards is
+ # that the server answered every caller rather than wedging.
+ assert all(c is not None for c in codes), (
+ f"{codes.count(None)}/{CONCURRENCY_N} API Gateway requests got no response at all — "
+ f"the Lambda dispatch wedged"
+ )
diff --git a/tests/test_dsql.py b/tests/test_dsql.py
index e8b324e0..3cad769b 100644
--- a/tests/test_dsql.py
+++ b/tests/test_dsql.py
@@ -4,17 +4,26 @@ validator unit tests and live data-plane (wire proxy) tests.
"""
import asyncio
+import os
import re
import socket
import struct
import threading
import time
+from urllib.parse import urlparse
import pytest
from botocore.exceptions import ClientError
from ministack.core import pgproxy
+# Raw HTTP calls here must hit the same server the boto3 fixtures use. Hardcoding
+# 4566 silently fails everywhere except a default-port run (CI), which is exactly
+# the kind of test that looks green until someone runs the suite on another port.
+_ENDPOINT = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566").rstrip("/")
+_ENDPOINT_HOST = urlparse(_ENDPOINT).hostname or "localhost"
+_ENDPOINT_PORT = urlparse(_ENDPOINT).port or 4566
+
ID_RE = re.compile(r"^[a-z0-9]{26}$")
@@ -244,7 +253,7 @@ class TestClusterLifecycle:
for bad in (0, 101):
req = urllib.request.Request(
- f"http://127.0.0.1:4566/cluster?max-results={bad}",
+ f"{_ENDPOINT}/cluster?max-results={bad}",
headers={"Host": "dsql.us-east-1.localhost"},
)
with pytest.raises(urllib.error.HTTPError) as exc:
diff --git a/tests/test_ecs.py b/tests/test_ecs.py
index af2f2008..49f0eb74 100644
--- a/tests/test_ecs.py
+++ b/tests/test_ecs.py
@@ -1,15 +1,14 @@
import copy
-import io
import json
import os
import time
import uuid as _uuid_mod
-import zipfile
from types import SimpleNamespace
from urllib.parse import urlparse
import pytest
from botocore.exceptions import ClientError
+from conftest import LoopProbe
from ministack.services import ecs as ecs_service
@@ -454,7 +453,7 @@ def test_ecs_create_service_v2(ecs):
ecs.create_cluster(clusterName="ecs-svc-v2c")
ecs.register_task_definition(
family="ecs-svc-v2td",
- containerDefinitions=[{"name": "w", "image": "nginx", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "w", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],
)
resp = ecs.create_service(
cluster="ecs-svc-v2c",
@@ -471,7 +470,7 @@ def test_ecs_describe_services_v2(ecs):
ecs.create_cluster(clusterName="ecs-ds-v2c")
ecs.register_task_definition(
family="ecs-ds-v2td",
- containerDefinitions=[{"name": "w", "image": "nginx", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "w", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],
)
ecs.create_service(
cluster="ecs-ds-v2c",
@@ -495,7 +494,7 @@ def test_ecs_update_service_v2(ecs):
ecs.create_cluster(clusterName="ecs-us-v2c")
ecs.register_task_definition(
family="ecs-us-v2td",
- containerDefinitions=[{"name": "w", "image": "nginx", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "w", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],
)
ecs.create_service(
cluster="ecs-us-v2c",
@@ -567,7 +566,7 @@ def test_ecs_service_arn_parser_does_not_tail_resolve_invalid_arns(ecs):
cluster_arn = ecs.create_cluster(clusterName=cluster)["cluster"]["clusterArn"]
ecs.register_task_definition(
family="ecs-arn-service-td",
- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],
)
created = ecs.create_service(
cluster=cluster,
@@ -656,7 +655,7 @@ def test_ecs_service_arn_parser_does_not_tail_resolve_invalid_arns(ecs):
def test_ecs_task_definition_arn_parser_does_not_tail_resolve_invalid_arns(ecs):
resp = ecs.register_task_definition(
family="ecs-arn-td",
- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],
)
task_definition_arn = resp["taskDefinition"]["taskDefinitionArn"]
valid = ecs.describe_task_definition(taskDefinition=task_definition_arn)
@@ -828,7 +827,7 @@ def test_ecs_timestamps_are_epoch(ecs):
# registeredAt might not be present on cluster, test on task def
ecs.register_task_definition(
family="ts-td-v44",
- containerDefinitions=[{"name": "app", "image": "nginx", "memory": 256}],
+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "memory": 256}],
)
td = ecs.describe_task_definition(taskDefinition="ts-td-v44")
registered_at = td["taskDefinition"].get("registeredAt")
@@ -847,7 +846,7 @@ def test_ecs_service_spawns_tasks(ecs):
ecs.create_cluster(clusterName=cluster)
ecs.register_task_definition(
family="svc-spawn-td",
- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],
)
ecs.create_service(
cluster=cluster,
@@ -872,7 +871,7 @@ def test_ecs_list_services(ecs):
ecs.create_cluster(clusterName=cluster)
ecs.register_task_definition(
family="ls-svc-td",
- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],
)
ecs.create_service(
cluster=cluster, serviceName="ls-svc-a", taskDefinition="ls-svc-td", desiredCount=1,
@@ -893,7 +892,7 @@ def test_ecs_service_running_count(ecs):
ecs.create_cluster(clusterName=cluster)
ecs.register_task_definition(
family="rc-td",
- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],
)
ecs.create_service(
cluster=cluster, serviceName="rc-svc", taskDefinition="rc-td", desiredCount=3,
@@ -910,7 +909,7 @@ def test_ecs_service_scale_up(ecs):
ecs.create_cluster(clusterName=cluster)
ecs.register_task_definition(
family="su-td",
- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],
)
ecs.create_service(
cluster=cluster, serviceName="su-svc", taskDefinition="su-td", desiredCount=1,
@@ -932,7 +931,7 @@ def test_ecs_service_scale_down(ecs):
ecs.create_cluster(clusterName=cluster)
ecs.register_task_definition(
family="sd-td",
- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],
)
ecs.create_service(
cluster=cluster, serviceName="sd-svc", taskDefinition="sd-td", desiredCount=3,
@@ -997,7 +996,7 @@ def test_ecs_service_delete_stops_tasks(ecs):
ecs.create_cluster(clusterName=cluster)
ecs.register_task_definition(
family="del-td",
- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],
)
ecs.create_service(
cluster=cluster, serviceName="del-svc", taskDefinition="del-td", desiredCount=2,
@@ -1027,7 +1026,7 @@ def test_ecs_service_scale_to_zero(ecs):
ecs.create_cluster(clusterName=cluster)
ecs.register_task_definition(
family="z-td",
- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],
)
ecs.create_service(
cluster=cluster, serviceName="z-svc", taskDefinition="z-td", desiredCount=2,
@@ -1050,7 +1049,7 @@ def test_ecs_cluster_task_counts(ecs):
ecs.create_cluster(clusterName=cluster)
ecs.register_task_definition(
family="ct-td",
- containerDefinitions=[{"name": "app", "image": "nginx", "cpu": 64, "memory": 128}],
+ containerDefinitions=[{"name": "app", "image": "alpine:latest", "command": ["sleep", "3600"], "cpu": 64, "memory": 128}],
)
ecs.create_service(
cluster=cluster, serviceName="ct-svc", taskDefinition="ct-td", desiredCount=3,
@@ -1364,3 +1363,37 @@ def test_ecs_container_secret_arn_selects_the_requested_region():
sm_service.reset()
set_request_account_id(original_account)
set_request_region(original_region)
+
+
+# ---------------------------------------------------------------------------
+# Re-entrancy under concurrency — the rest of the suite issues one request at
+# a time, so a service whose blocking work is misclassified passes serially
+# and only wedges under load. These fire N callers at once and assert both
+# that every caller completes and that the event loop keeps serving.
+# ---------------------------------------------------------------------------
+def test_ecs_run_task_does_not_block_the_loop(ecs):
+ """RunTask talks to the Docker daemon; that must never happen on the loop."""
+ cluster = f"conc-{_uuid_mod.uuid4().hex[:8]}"
+ ecs.create_cluster(clusterName=cluster)
+ ecs.register_task_definition(
+ family=f"{cluster}-td",
+ containerDefinitions=[{"name": "app", "image": "alpine:latest",
+ "command": ["sleep", "3600"], "memory": 64, "essential": True}],
+ )
+ with LoopProbe() as probe:
+ try:
+ ecs.run_task(cluster=cluster, taskDefinition=f"{cluster}-td", count=1)
+ except Exception as exc: # no daemon / image pull refused
+ pytest.skip(f"ECS RunTask unavailable in this environment: {exc}")
+ finally:
+ for arn in ecs.list_tasks(cluster=cluster).get("taskArns", []):
+ try:
+ ecs.stop_task(cluster=cluster, task=arn)
+ except Exception:
+ pass
+ try:
+ ecs.delete_cluster(cluster=cluster)
+ except Exception:
+ pass
+ probe.assert_responsive("ECS RunTask")
diff --git a/tests/test_eks.py b/tests/test_eks.py
index 6a4fb730..812a656a 100644
--- a/tests/test_eks.py
+++ b/tests/test_eks.py
@@ -645,11 +645,14 @@ def test_eks_k3s_run_kwargs_container_name_and_labels_are_region_scoped():
assert east["name"] == "ministack-eks-us-east-1-my-cluster"
assert west["name"] == "ministack-eks-us-west-2-my-cluster"
- assert east["labels"] == {
+ # Subset, not equality: containers also carry the ownership labels
+ # (`ministack.instance` / `ministack.boot`) that scope reaping to this
+ # MiniStack. This test is about region scoping, so it asserts only that.
+ assert {
"ministack": "eks",
"cluster_name": "my-cluster",
"region": "us-east-1",
- }
+ }.items() <= east["labels"].items()
assert west["labels"]["region"] == "us-west-2"
diff --git a/tests/test_elasticache.py b/tests/test_elasticache.py
index bdeb136c..a27cb590 100644
--- a/tests/test_elasticache.py
+++ b/tests/test_elasticache.py
@@ -1732,3 +1732,103 @@ def test_valkey_no_docker_fallback_uses_redis_port(monkeypatch):
)
assert (host, port) == (elasticache.REDIS_DEFAULT_HOST, elasticache.REDIS_DEFAULT_PORT)
assert cid is None
+
+
+# ---------------------------------------------------------------------------
+# Deferred (cold-image) provisioning — state, not liveness
+# ---------------------------------------------------------------------------
+
+def test_elasticache_cold_image_publishes_the_real_endpoint(fake_docker, monkeypatch):
+ """Describe must report the port the container actually listens on.
+
+ When the engine image is not cached, CreateCacheCluster returns at once and
+ a background finisher starts the container. The finisher wrote
+ `ConfigurationEndpoint` while every Describe path reads `_endpoint` and
+ `CacheNodes[].Endpoint` — which create had filled with the *engine default*
+ port, no container existing yet. So a cold-image create reported
+ `localhost:6379` for the rest of its life while the container was on the
+ allocated host port, and never self-corrected. Reported by @Areson.
+ """
+ import threading
+
+ from ministack.core.responses import set_request_account_id, set_request_region
+ from ministack.services import elasticache as ec
+
+ set_request_account_id("000000000000")
+ set_request_region("us-east-1")
+ monkeypatch.setattr(ec, "_get_docker", lambda: fake_docker)
+ monkeypatch.setattr(ec, "_image_is_local", lambda *_a: False)
+ monkeypatch.setattr(ec, "_spawn_redis_container",
+ lambda **kw: ("localhost", 16379, "cid-real"))
+
+ done = threading.Event()
+ orig = ec.spawn_background
+
+ def tracked(fn, *a, **kw):
+ t = orig(fn, *a, **kw)
+ threading.Thread(target=lambda: (t.join(), done.set()), daemon=True).start()
+ return t
+ monkeypatch.setattr(ec, "spawn_background", tracked)
+
+ ec._create_cache_cluster({"CacheClusterId": ["cold-1"], "Engine": ["redis"],
+ "CacheNodeType": ["cache.t3.micro"], "NumCacheNodes": ["1"]})
+ assert done.wait(timeout=10), "deferred finisher never completed"
+
+ rec = ec._clusters["cold-1"]
+ assert rec["CacheClusterStatus"] == "available"
+ assert rec["_endpoint"] == {"Address": "localhost", "Port": 16379}, (
+ f"Describe would report {rec['_endpoint']} — nothing is listening there"
+ )
+ assert rec["CacheNodes"][0]["Endpoint"]["Port"] == 16379
+
+
+def test_elasticache_delete_during_deferred_start_leaves_no_container(fake_docker, monkeypatch):
+ """A delete while the container is being started must not strand it.
+
+ The finisher checked the record existed and then published the container id
+ into it. A delete in that gap removed the record, found no id to tear down,
+ and the container ran on referenced by nothing — unreachable by any API
+ call. Reported by @Areson.
+ """
+ import threading
+
+ from ministack.core.responses import set_request_account_id, set_request_region
+ from ministack.services import elasticache as ec
+
+ set_request_account_id("000000000000")
+ set_request_region("us-east-1")
+ monkeypatch.setattr(ec, "_get_docker", lambda: fake_docker)
+ monkeypatch.setattr(ec, "_image_is_local", lambda *_a: False)
+
+ spawned, release, finished = threading.Event(), threading.Event(), threading.Event()
+
+ def fake_spawn(**kw):
+ name = kw.get("name")
+ fake_docker.containers.run(name=name, labels=kw.get("labels"))
+ spawned.set()
+ release.wait(timeout=10) # hold inside the provisioning window
+ return "localhost", 16380, f"cid-{name}"
+ monkeypatch.setattr(ec, "_spawn_redis_container", fake_spawn)
+
+ orig = ec.spawn_background
+
+ def tracked(fn, *a, **kw):
+ t = orig(fn, *a, **kw)
+ threading.Thread(target=lambda: (t.join(), finished.set()), daemon=True).start()
+ return t
+ monkeypatch.setattr(ec, "spawn_background", tracked)
+
+ ec._create_cache_cluster({"CacheClusterId": ["race-1"], "Engine": ["redis"],
+ "CacheNodeType": ["cache.t3.micro"], "NumCacheNodes": ["1"]})
+ assert spawned.wait(timeout=10), "container was never started"
+
+ ec._delete_cache_cluster({"CacheClusterId": ["race-1"]})
+ release.set()
+ assert finished.wait(timeout=10), "finisher never completed"
+
+ assert "race-1" not in ec._clusters
+ assert fake_docker.live() == [], (
+ f"leaked container(s) {fake_docker.live()} — running, and no API call can reach them"
+ )
diff --git a/tests/test_lambda.py b/tests/test_lambda.py
index 6e901f44..74de1769 100644
--- a/tests/test_lambda.py
+++ b/tests/test_lambda.py
@@ -15,6 +15,13 @@ import boto3
import pytest
from botocore.config import Config
from botocore.exceptions import ClientError
+from conftest import (
+ CONCURRENCY_N_NESTED,
+ ENDPOINT,
+ LoopProbe,
+ concurrent_burst,
+ make_probe_lambda,
+)
_endpoint = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566").rstrip("/")
@@ -1657,7 +1664,6 @@ def test_lambda_esm_sqs_report_batch_item_failures(lam, sqs):
def test_lambda_warm_start(lam, apigw):
"""Warm worker via API Gateway execute-api: module-level state persists across invocations."""
import urllib.request as _urlreq
- import uuid as _uuid
fname = f"intg-warm-{_uuid_mod.uuid4().hex[:8]}"
code = (
@@ -2768,7 +2774,6 @@ def test_lambda_invoke_dry_run_returns_204(lam):
assert resp["StatusCode"] == 204
def test_lambda_layer_publish(lam):
- import base64
import io
import zipfile
@@ -2827,7 +2832,6 @@ def test_lambda_layer_list_layers(lam):
assert "my-test-layer" in names
def test_lambda_layer_delete_version(lam):
- import base64
import io
import zipfile
@@ -3052,7 +3056,7 @@ def test_lambda_pool_kill_function_reaps_all_qualifiers():
self.removed = False
def stop(self, timeout=2):
self.stopped = True
- def remove(self, force=False):
+ def remove(self, force=False, v=False):
self.removed = True
stubs = [_StubContainer() for _ in range(3)]
@@ -4042,7 +4046,6 @@ def test_lambda_provided_runtime_parallel_invocations():
def test_apigwv2_nodejs_lambda_proxy(lam, apigw):
"""API Gateway v2 HTTP API should invoke Node.js Lambda via warm worker, not return mock."""
import urllib.request as _urlreq
- import uuid as _uuid
from botocore.exceptions import ClientError
@@ -7587,7 +7590,6 @@ def test_nodejs_worker_aws_sdk_v3_stub_wire_roundtrip(lam, ssm):
router.py undetected by the resolution-only tests).
"""
import shutil
- import uuid as _uuid
if not shutil.which("node"):
pytest.skip("node not found on PATH")
@@ -7696,7 +7698,6 @@ def _create_durable_execution_directly(lam):
response header. Returns (function_name, function_arn, dict with
DurableExecutionArn + CheckpointToken)."""
import base64 as _b64
- import json as _json
fname = f"durable-fn-{_uuid_mod.uuid4().hex[:8]}"
try:
lam.delete_function(FunctionName=fname)
@@ -7738,7 +7739,6 @@ def test_lambda_durable_function_config_round_trip(lam):
except Exception:
pass
import base64 as _b64
- import json as _json
zip_b64 = _b64.b64encode(_make_zip("def handler(e,c): return e")).decode()
code, body = _raw_durable("POST", "/2015-03-31/functions", body={
"FunctionName": fname,
@@ -7870,7 +7870,6 @@ def test_lambda_durable_stop(lam):
def test_lambda_durable_list_by_function(lam):
fname, fn_arn, rec = _create_durable_execution_directly(lam)
try:
- from urllib.parse import quote
code, body = _raw_durable("GET", f"/2025-12-01/functions/{fname}/durable-executions")
assert code == 200
arns = [s["DurableExecutionArn"] for s in body["DurableExecutions"]]
@@ -7940,7 +7939,6 @@ def test_lambda_durable_chained_invoke_runs_child(lam):
the child function and records the result back into the parent's
operation log."""
import base64 as _b64
- import json as _json
parent = f"durable-parent-{_uuid_mod.uuid4().hex[:8]}"
child = f"durable-child-{_uuid_mod.uuid4().hex[:8]}"
for n in (parent, child):
@@ -8881,7 +8879,6 @@ def test_lambda_durable_create_function_durable_config_round_trip_with_update(la
"""DurableConfig must survive UpdateFunctionConfiguration that touches
unrelated fields (timeout, memory)."""
import base64 as _b64
- import json as _json
fname = f"dur-upd-{_uuid_mod.uuid4().hex[:8]}"
try:
@@ -10110,7 +10107,7 @@ def test_lambda_keepalive_zero_forces_cold_start(monkeypatch):
pass
def stop(self, timeout=2):
self.stopped = True
- def remove(self, force=False):
+ def remove(self, force=False, v=False):
self.removed = True
stub = _StubContainer()
@@ -10139,6 +10136,128 @@ def test_lambda_keepalive_zero_forces_cold_start(monkeypatch):
# ---------------------------------------------------------------------------
+# Re-entrant invocation — a handler that calls back into the emulator during
+# its own invocation, including into its own function.
+#
+# Every synchronous execution used to hold one worker out of the event loop's
+# bounded default pool for the whole handler duration, so a handler's nested
+# call queued behind the invocations waiting on it and wedged at exactly the
+# pool size. Separately, the local executor kept a single Worker per function
+# whose lock was held for the whole handler, so a function invoking *itself*
+# deadlocked on the first hop regardless of threading.
+#
+# Executions now run on a dedicated thread each (core.concurrency.run_reentrant)
+# and draw a warm worker from a per-function pool, with AWS's concurrency model
+# enforced once, in lambda_svc._acquire_execution_slot.
+# ---------------------------------------------------------------------------
+
+_SELF_INVOKE_CODE = """
+import json, os, urllib.request
+
+def handler(event, context):
+ depth = int(event.get("depth", 0))
+ limit = int(event.get("limit", 3))
+ if depth >= limit:
+ return {"depth": depth, "leaf": True}
+ req = urllib.request.Request(
+ os.environ["AWS_ENDPOINT_URL"]
+ + "/2015-03-31/functions/" + context.function_name + "/invocations",
+ data=json.dumps({"depth": depth + 1, "limit": limit}).encode(),
+ headers={"Content-Type": "application/json"},
+ )
+ with urllib.request.urlopen(req, timeout=25) as resp:
+ return {"depth": depth, "child": json.loads(resp.read().decode())}
+"""
+
+
+ os.environ.get("LAMBDA_EXECUTOR", "").lower() == "docker",
+ reason="exercises the in-process warm worker pool; the docker executor "
+ "spawns a container per concurrent invocation and has no shared lock",
+)
+def test_lambda_self_invoking_handler_does_not_deadlock(lam):
+ """A handler that invokes its own function completes.
+
+ Real AWS allows this and only intervenes at the ~16-hop recursive-loop
+ limit. Before the per-function worker pool this deadlocked at the *first*
+ hop, with a single caller and no concurrency involved: the nested invoke
+ waited on the worker lock its own parent held, and the invocation burned
+ its whole timeout.
+ """
+ name = f"lam-self-invoke-{_uuid_mod.uuid4().hex[:8]}"
+ lam.create_function(
+ FunctionName=name, Runtime="python3.12", Handler="index.handler",
+ Role=_LAMBDA_ROLE, Code={"ZipFile": _make_zip(_SELF_INVOKE_CODE)},
+ Timeout=30,
+ )
+ try:
+ resp = lam.invoke(
+ FunctionName=name,
+ Payload=json.dumps({"depth": 0, "limit": 4}).encode(),
+ )
+ assert resp.get("FunctionError", "") == "", resp.get("FunctionError")
+ payload = json.loads(resp["Payload"].read())
+ # Four nested levels, each reporting its own depth, innermost a leaf.
+ for expected in range(4):
+ assert payload["depth"] == expected
+ payload = payload["child"]
+ assert payload == {"depth": 4, "leaf": True}
+ finally:
+ with contextlib.suppress(Exception):
+ lam.delete_function(FunctionName=name)
+
+
+def test_lambda_reserved_concurrency_throttles_rather_than_queues(lam):
+ """At ReservedConcurrentExecutions the extra invocation is throttled.
+
+ AWS returns TooManyRequestsException rather than queueing. Queueing would
+ also reintroduce the deadlock: an invocation waiting for a unit its own
+ caller holds waits forever.
+ """
+ import concurrent.futures
+
+ name = f"lam-reserved-{_uuid_mod.uuid4().hex[:8]}"
+ code = "import time\ndef handler(event, context):\n time.sleep(1.5)\n return {'ok': True}\n"
+ lam.create_function(
+ FunctionName=name, Runtime="python3.12", Handler="index.handler",
+ Role=_LAMBDA_ROLE, Code={"ZipFile": _make_zip(code)}, Timeout=30,
+ )
+ try:
+ lam.put_function_concurrency(
+ FunctionName=name, ReservedConcurrentExecutions=1)
+
+ def _invoke(_i):
+ try:
+ resp = lam.invoke(FunctionName=name, Payload=b"{}")
+ resp["Payload"].read()
+ return resp["StatusCode"], "", ""
+ except ClientError as exc:
+ return (exc.response["ResponseMetadata"]["HTTPStatusCode"],
+ exc.response["Error"]["Code"],
+ exc.response.get("Reason", ""))
+
+ with concurrent.futures.ThreadPoolExecutor(max_workers=3) as ex:
+ results = list(ex.map(_invoke, range(3)))
+
+ throttled = [r for r in results if r[0] == 429]
+ succeeded = [r for r in results if r[0] == 200 and not r[1]]
+ assert succeeded, f"reservation of 1 must still admit one invoke: {results}"
+ assert throttled, (
+ f"invocations beyond ReservedConcurrentExecutions=1 must throttle, "
+ f"not queue or hang: {results}"
+ )
+ # The code, not just the status. AWS models exactly one throttle shape
+ # for Invoke; anything else is a string AWS never sends, and
+ # ``except lambda.exceptions.TooManyRequestsException`` would not catch
+ # it — a 429 alone passes on the wrong shape.
+ for status, code, reason in throttled:
+ assert code == "TooManyRequestsException", (
+ f"expected TooManyRequestsException, got {code!r}")
+ assert reason == "ReservedFunctionConcurrentInvocationLimitExceeded", (
+ f"expected the function-limit Reason, got {reason!r}")
+ finally:
+ with contextlib.suppress(Exception):
+ lam.delete_function(FunctionName=name)
# Recursive-loop detection — PutFunctionRecursionConfig made real.
#
# AWS drops an invocation once a request has gone around the same function
@@ -10430,3 +10549,90 @@ def test_lambda_inbound_invoke_depth_parsing(headers, expected):
from ministack.services.lambda_svc import _inbound_invoke_depth
assert _inbound_invoke_depth(headers) == expected
+
+
+# ---------------------------------------------------------------------------
+# Re-entrancy under concurrency — the rest of the suite issues one request at
+# a time, so a service whose blocking work is misclassified passes serially
+# and only wedges under load. These fire N callers at once and assert both
+# that every caller completes and that the event loop keeps serving.
+# ---------------------------------------------------------------------------
+def test_concurrent_nested_lambda_invocations_do_not_starve(lam):
+ """Callers that each invoke another function must never wedge.
+
+ Three details make this discriminate; without any one of them a build with
+ the bug still passes:
+
+ - **distinct caller functions.** One function invoked N times serialises on
+ its own worker lock and never reaches the shared pool.
+ - **a hold before the nested call**, so every caller is in flight at once
+ rather than completing before the next starts.
+ - **N above the pool size**, since starvation begins at exactly the pool
+ bound.
+
+ A throttle is a correct answer and is allowed — the account concurrency cap
+ is deliberately below N on small hosts. What must never happen is a caller
+ getting no answer at all, which is what starvation looks like.
+ """
+ callee = make_probe_lambda(lam, "def handler(event, context):\n return {'pong': True}\n")
+ caller_src = f"""
+import json, os, time, urllib.request
+
+def handler(event, context):
+ time.sleep(float(event.get("hold", 1.5)))
+ ep = os.environ["AWS_ENDPOINT_URL"]
+ req = urllib.request.Request(
+ ep + "/2015-03-31/functions/{callee}/invocations",
+ data=json.dumps({{}}).encode(),
+ headers={{"Content-Type": "application/json"}},
+ )
+ return {{"nested": json.loads(urllib.request.urlopen(req, timeout=25).read().decode())}}
+"""
+ callers = [make_probe_lambda(lam, caller_src, timeout=40) for _ in range(CONCURRENCY_N_NESTED)]
+
+ # A client sized for the burst. The shared fixture client pools far fewer
+ # connections than we fire, and the resulting client-side queueing shows up
+ # as latency that looks like server stall but is not.
+ import boto3
+ from botocore.config import Config as _Config
+
+ burst_client = boto3.client(
+ "lambda", endpoint_url=ENDPOINT, aws_access_key_id="test",
+ aws_secret_access_key="test", region_name=os.environ.get("AWS_DEFAULT_REGION", "us-east-1"),
+ config=_Config(retries={"max_attempts": 0}, read_timeout=120,
+ max_pool_connections=CONCURRENCY_N_NESTED + 8),
+ )
+
+ def invoke(name, hold=1.5):
+ try:
+ payload = burst_client.invoke(
+ FunctionName=name, Payload=json.dumps({"hold": hold}).encode())
+ body = payload["Payload"].read().decode() or "{}"
+ if payload.get("FunctionError"):
+ # A timeout is the starvation signature: the nested call never
+ # got a slot. Anything else is the handler's own failure.
+ return "wedged" if "timed out" in body.lower() else "error"
+ return "ok"
+ except Exception as exc:
+ return "throttled" if "TooManyRequests" in str(exc) else "wedged"
+
+ # Warm every caller first, in small batches. Otherwise the measured burst
+ # pays 70 simultaneous cold starts, which a 2-core CI runner cannot finish
+ # inside the function timeout — 16 of 70 timed out that way, which reads as
+ # starvation but is only the box running out of CPU. Warm workers make the
+ # burst measure scheduling, which is what this test is about.
+ for i in range(0, len(callers), 10):
+ concurrent_burst(lambda n: invoke(n, hold=0.0), items=callers[i:i + 10])
+
+ with LoopProbe() as probe:
+ results = concurrent_burst(invoke, items=callers)
+ probe.assert_responsive("nested lambda burst")
+
+ wedged = results.count("wedged")
+ assert wedged == 0, (
+ f"{wedged}/{CONCURRENCY_N_NESTED} nested invocations never completed — re-entrant work is "
+ f"queueing behind itself (ok={results.count('ok')}, "
+ f"throttled={results.count('throttled')}, error={results.count('error')})"
+ )
+ assert results.count("ok") > 0, "no invocation succeeded at all"
diff --git a/tests/test_ministack.py b/tests/test_ministack.py
index d5bbd73f..8cc6fabe 100644
--- a/tests/test_ministack.py
+++ b/tests/test_ministack.py
@@ -733,3 +733,81 @@ class TestExtractS3VhostBucket:
def test_bare_s3_without_region(self):
assert _extract_s3_vhost_bucket("s3.localhost") is None
+
+
+# ---------------------------------------------------------------------------
+# Container reaping is scoped to this MiniStack instance
+# ---------------------------------------------------------------------------
+
+def _exited_container(daemon, reaper, name, **extra):
+ c = daemon.containers.run(name=name, labels=reaper.own_labels("rds", **extra))
+ c.status = "exited"
+ c.attrs["State"] = {"FinishedAt": ""}
+ return c
+
+
+def test_reaper_never_touches_another_instances_container(fake_docker, monkeypatch):
+ """A second gateway on the same daemon must not reclaim the first's work.
+
+ The reaper reclaimed any exited ministack-labelled container absent from
+ *this* process's records. With several instances on one Docker daemon that
+ is wrong: `StopDBCluster` leaves an exited container on purpose, expecting a
+ later start, and a second gateway has no record of it — so it was reaped
+ after the grace period, destroying state its owner meant to keep. Observed
+ live by @Areson, one instance removing another's container ~3 minutes after
+ it was stopped.
+ """
+ from ministack.core import container_reaper
+
+ monkeypatch.setattr(container_reaper, "EXITED_GRACE", 0)
+ monkeypatch.setenv("GATEWAY_PORT", "4566")
+ _exited_container(fake_docker, container_reaper, "owned-by-A", db_id="a")
+
+ monkeypatch.setenv("GATEWAY_PORT", "4599") # a different instance
+ monkeypatch.setattr(container_reaper, "_boot_nonce", "nonce-B")
+ container_reaper.register_live_ids("rds", lambda: [])
+
+ assert container_reaper.reap_abandoned(fake_docker) == 0
+ assert "owned-by-A" in fake_docker.live(), "instance B destroyed instance A's container"
+
+
+def test_reaper_still_reclaims_its_own_abandoned_container(fake_docker, monkeypatch):
+ """Instance scoping must not cost the reclamation it was added for."""
+ from ministack.core import container_reaper
+
+ monkeypatch.setattr(container_reaper, "EXITED_GRACE", 0)
+ monkeypatch.setenv("GATEWAY_PORT", "4566")
+ monkeypatch.setattr(container_reaper, "_boot_nonce", "nonce-mine")
+ _exited_container(fake_docker, container_reaper, "owned-by-me", db_id="mine")
+ container_reaper.register_live_ids("rds", lambda: [])
+
+ assert container_reaper.reap_abandoned(fake_docker) == 1
+ assert fake_docker.live() == []
+
+
+def test_boot_sweep_takes_our_predecessor_but_spares_other_instances(fake_docker, monkeypatch):
+ """Boot reclaims the previous run on our port, never another instance.
+
+ Holding the gateway port proves the previous owner of that address is gone,
+ so its leftovers are ours to clean; a container labelled for a different
+ address belongs to a MiniStack that may still be running.
+ """
+ from ministack.core import container_reaper
+
+ monkeypatch.setenv("GATEWAY_PORT", "4566")
+ monkeypatch.setattr(container_reaper, "_boot_nonce", "run-1")
+ _exited_container(fake_docker, container_reaper, "ours-previous-run", db_id="old")
+
+ monkeypatch.setenv("GATEWAY_PORT", "4777")
+ monkeypatch.setattr(container_reaper, "_boot_nonce", "other")
+ _exited_container(fake_docker, container_reaper, "another-instance", db_id="theirs")
+
+ monkeypatch.setenv("GATEWAY_PORT", "4566") # we restart on our port
+ monkeypatch.setattr(container_reaper, "_boot_nonce", "run-2")
+ container_reaper.reap_all(fake_docker, include_unlabelled=True)
+
+ assert "ours-previous-run" not in fake_docker.live(), "boot sweep left our own orphan"
+ assert "another-instance" in fake_docker.live(), "boot sweep destroyed another instance's container"
diff --git a/tests/test_mwaa.py b/tests/test_mwaa.py
index c1cae46a..adfc799c 100644
--- a/tests/test_mwaa.py
+++ b/tests/test_mwaa.py
@@ -319,13 +319,18 @@ def test_airflow_runtime_identity_and_background_scope_include_region(
assert dag_sync_complete.wait(timeout=2)
assert docker_client.containers.get_calls == [expected_name, legacy_name]
- assert legacy_container.remove_calls == [{"force": True}]
+ # v=True reclaims the container's anonymous volume. Verified against
+ # docker: named volumes (MWAA's dags/db persistence) are untouched
+ # by remove(v=True), so this cannot lose persisted state.
+ assert legacy_container.remove_calls == [{"force": True, "v": True}]
assert docker_client.containers.run_kwargs["name"] == expected_name
- assert docker_client.containers.run_kwargs["labels"] == {
+ # Subset: ownership labels (`ministack.instance` / `ministack.boot`)
+ # are also present and are not what this test is about.
+ assert {
"ministack": "mwaa",
"region": region,
"env_name": env_name,
- }
+ }.items() <= docker_client.containers.run_kwargs["labels"].items()
expected_volume_prefix = legacy_name if legacy_volumes else expected_name
assert set(docker_client.containers.run_kwargs["volumes"]) == {
f"{expected_volume_prefix}-dags",
@@ -356,7 +361,7 @@ def test_reset_stops_containers_in_every_region(monkeypatch):
def stop(self, timeout):
self.stop_calls.append(timeout)
- def remove(self, force):
+ def remove(self, force=False, v=False):
self.remove_calls.append(force)
class FakeContainers:
diff --git a/tests/test_opensearch.py b/tests/test_opensearch.py
index 4225509b..053e38fd 100644
--- a/tests/test_opensearch.py
+++ b/tests/test_opensearch.py
@@ -672,3 +672,44 @@ def test_opensearch_dataplane_names_include_region():
)
finally:
set_request_region(original_region)
+
+
+def test_opensearch_cold_image_publishes_the_real_endpoint(fake_docker, monkeypatch):
+ """`_Endpoint` is what readers prefer, so the finisher must update it.
+
+ Same defect @Areson reported against ElastiCache, found here by inventory:
+ create left `_Endpoint` at the `localhost:BASE_PORT` placeholder and the
+ deferred finisher updated only `Endpoint`, so Describe kept reporting an
+ address nothing listens on.
+ """
+ import threading
+
+ from ministack.core.responses import set_request_account_id, set_request_region
+ from ministack.services import opensearch as os_
+
+ set_request_account_id("000000000000")
+ set_request_region("us-east-1")
+ monkeypatch.setattr(os_, "_get_docker", lambda: fake_docker, raising=False)
+ monkeypatch.setattr(os_, "_image_is_local", lambda *_a: False, raising=False)
+ monkeypatch.setattr(os_, "DATAPLANE_ENABLED", True, raising=False)
+ monkeypatch.setattr(os_, "_spawn_dataplane",
+ lambda name, ver: ("localhost", 19200, "cid-os", None, None))
+
+ done = threading.Event()
+ orig = os_.spawn_background
+
+ def tracked(fn, *a, **kw):
+ t = orig(fn, *a, **kw)
+ threading.Thread(target=lambda: (t.join(), done.set()), daemon=True).start()
+ return t
+ monkeypatch.setattr(os_, "spawn_background", tracked)
+
+ os_.create_domain_record({"DomainName": "cold-dom"})
+ assert done.wait(timeout=10), "deferred finisher never completed"
+
+ rec = os_._domains["cold-dom"]
+ assert rec["_Endpoint"] == "localhost:19200", (
+ f"readers prefer _Endpoint and it still says {rec['_Endpoint']!r}"
+ )
+ assert rec["Endpoint"] == "localhost:19200"
diff --git a/tests/test_rds.py b/tests/test_rds.py
index a8bdaf1b..5cbcee9a 100644
--- a/tests/test_rds.py
+++ b/tests/test_rds.py
@@ -4931,12 +4931,14 @@ def test_rds_restore_state_respawns_docker_container(monkeypatch):
assert runs[0]["environment"]["POSTGRES_USER"] == "admin"
assert runs[0]["environment"]["POSTGRES_PASSWORD"] == "password123"
assert runs[0]["environment"]["POSTGRES_DB"] == "mydb"
- assert runs[0]["labels"] == {
+ # Subset: ownership labels (`ministack.instance` / `ministack.boot`) are
+ # also stamped so the reaper cannot cross instance boundaries.
+ assert {
"ministack": "rds",
"db_id": db_id,
"account_id": get_account_id(),
"region": get_region(),
- }
+ }.items() <= runs[0]["labels"].items()
restored = m._instances.get(db_id)
assert restored is not None
@@ -5251,13 +5253,22 @@ def test_rds_restore_state_respawns_one_container_per_cluster(
assert runs == []
release_migration_remove.set()
- deadline = time.time() + 2
+ # restore_state provisions on a background thread, so everything below
+ # depends on that thread having finished. The wait used to be a bare 2 s
+ # budget that fell through silently when it expired, and the test then
+ # failed on whichever later assertion happened to notice — an empty
+ # `removed_volumes` diff, with no hint that the cause was a timeout. It
+ # now waits long enough for a loaded CI runner and says so when it gives
+ # up, so a slow box reads as slow rather than as a behaviour change.
+ deadline = time.time() + 20
+ settled = False
while time.time() < deadline:
if scenario in (
"last-member-deleted-during-readiness",
"last-members-deleted-before-start",
):
if m._clusters.get(cluster_id, {}).get("DBClusterMembers") == []:
+ settled = True
break
time.sleep(0.01)
continue
@@ -5266,8 +5277,14 @@ def test_rds_restore_state_respawns_one_container_per_cluster(
for db_id in ("restored-writer", "restored-reader")
}
if statuses <= {"available", "failed"}:
+ settled = True
break
time.sleep(0.01)
+ assert settled, (
+ f"{scenario}: restore did not settle within 20s — the background "
+ f"provisioning thread is still running, so the assertions below "
+ f"would report a timeout as a behaviour change"
+ )
if not writer_removal_succeeds or not reader_removal_succeeds or (
scenario == "ownership-mismatch"
diff --git a/tests/test_stepfunctions.py b/tests/test_stepfunctions.py
index e6e951ee..7c539273 100644
--- a/tests/test_stepfunctions.py
+++ b/tests/test_stepfunctions.py
@@ -11,7 +11,7 @@ import boto3
import pytest
from botocore.config import Config
from botocore.exceptions import ClientError
-from conftest import ENDPOINT
+from conftest import ENDPOINT, LoopProbe, concurrent_burst
def _make_zip(code: str) -> bytes:
@@ -7523,3 +7523,33 @@ def test_sfn_jsonata_format_number(sfn_sync):
"neg_sub": "(1,234.50)",
"plain": "1,234",
}
+
+
+# ---------------------------------------------------------------------------
+# Re-entrancy under concurrency — the rest of the suite issues one request at
+# a time, so a service whose blocking work is misclassified passes serially
+# and only wedges under load. These fire N callers at once and assert both
+# that every caller completes and that the event loop keeps serving.
+# ---------------------------------------------------------------------------
+def test_concurrent_sync_executions_do_not_block_the_loop(sfn, sfn_sync):
+ """StartSyncExecution runs off the loop; N at once must not stall the server."""
+ definition = json.dumps({
+ "StartAt": "Wait",
+ "States": {"Wait": {"Type": "Wait", "Seconds": 1, "Next": "Done"},
+ "Done": {"Type": "Pass", "End": True}},
+ })
+ arn = sfn.create_state_machine(
+ name=f"conc-sync-{_uuid_mod.uuid4().hex[:8]}", definition=definition,
+ roleArn="arn:aws:iam::000000000000:role/R",
+ )["stateMachineArn"]
+
+ def run(i):
+ return sfn_sync.start_sync_execution(
+ stateMachineArn=arn, input=json.dumps({"i": i}))["status"]
+
+ with LoopProbe() as probe:
+ statuses = concurrent_burst(run)
+ probe.assert_responsive("sync execution burst")
+
+ assert all(s == "SUCCEEDED" for s in statuses), f"sync executions returned {set(statuses)}"
diff --git a/tests/test_transfer.py b/tests/test_transfer.py
index 0db79888..d62979cf 100644
--- a/tests/test_transfer.py
+++ b/tests/test_transfer.py
@@ -323,7 +323,6 @@ def test_transfer_workos_sftp_workflow(transfer):
import asyncio
import os
import socket
-import time
import urllib.request
import uuid
@@ -791,6 +790,11 @@ def test_sftp_auth_uses_server_region_and_stashes_connection_metadata():
assert service._sftp_server_state(account_id, west_region, server_id) == "OFFLINE"
connection = _FakeConnection()
+ # `_MiniStackSSHServer` is built by `_ensure_sftp_runtime()`, not at
+ # import — reaching for the class directly is only safe once that has
+ # run. Previously this test passed only when some earlier test in the
+ # same process happened to trigger it.
+ service._ensure_sftp_runtime()
auth_server = service._MiniStackSSHServer()
auth_server.connection_made(connection)
assert auth_server.validate_public_key(