Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

Fix RDS Data API readiness race (#666)

Jay Janssena799e48affff
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b876c135..266216e0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -33,6 +33,10 @@ Versioning follows [Semantic Versioning](https://semver.org/).
### Changed
- **Test harness — xdist session-startup reset coordination** — the per-worker `autouse` `reset_server` fixture previously had every xdist worker hit `/_ministack/reset` on session start, so a slower worker's reset could fire after a faster worker had already begun creating fixtures, wiping that state mid-test (most visibly as occasional `list_functions()` empty results in `test_lambda_create_invoke`). The first worker now wins an `O_EXCL` file lock and runs the reset; the others wait briefly for a marker file and skip. Single-process pytest (no xdist) keeps the original behaviour. Removes a class of CI flakes that only reproduced under parallel load.
+### Fixed
+- **RDS Data API no longer acknowledges writes through the SQL stub for real containers that are still booting** — Docker-backed RDS instances now wait for an authenticated database connection before reporting `available`, and their parent clusters remain `creating` while member instances are not ready. If an endpoint-backed container still cannot be reached, RDS Data API returns `DatabaseUnavailableException` instead of silently tracking `CREATE USER` / `GRANT` statements in memory. Stub SQL mode remains available for control-plane-only clusters without a real backing container. Contributed by @jayjanssen.
+- **RDS Aurora MySQL local endpoint and master-user parity** — Aurora cluster endpoints now track the reachable backing DB instance endpoint after cluster members are created, so Lambda containers using `DescribeDBClusters.Endpoint` can connect to MiniStack-backed databases. MySQL and Aurora MySQL containers also grant the configured master username AWS/RDS-like global privileges after the server is reachable, with version-specific dynamic privileges treated as best-effort. Contributed by @jayjanssen.
+
---
## [1.3.41] — 2026-05-16
diff --git a/ministack/services/rds.py b/ministack/services/rds.py
index c7190e00..41561bd9 100644
--- a/ministack/services/rds.py
+++ b/ministack/services/rds.py
@@ -180,6 +180,111 @@ def _wait_for_port(host, port, timeout=60):
return False
+def _is_mysql_engine(engine):
+ return any(e in engine for e in ("mysql", "aurora-mysql", "mariadb"))
+
+
+def _is_postgres_engine(engine):
+ return any(e in engine for e in ("postgres", "aurora-postgresql"))
+
+
+def _grant_mysql_master_user_privileges(host, port, master_user, master_pass, db_id):
+ """Grant the emulated MySQL master user AWS/RDS-like admin privileges."""
+ try:
+ import pymysql
+ conn = pymysql.connect(
+ host=host, port=int(port), user="root",
+ password=master_pass, autocommit=True)
+ cur = conn.cursor()
+ cur.execute(
+ "CREATE USER IF NOT EXISTS %s@'%%' IDENTIFIED BY %s",
+ (master_user, master_pass),
+ )
+ cur.execute(
+ "GRANT ALL PRIVILEGES ON *.* TO %s@'%%' WITH GRANT OPTION",
+ (master_user,),
+ )
+ for privilege in ("APPLICATION_PASSWORD_ADMIN",):
+ try:
+ cur.execute(f"GRANT {privilege} ON *.* TO %s@'%%'", (master_user,))
+ except Exception as e:
+ logger.debug(
+ "RDS: MySQL privilege %s unsupported for %s: %s",
+ privilege, db_id, e)
+ cur.execute("FLUSH PRIVILEGES")
+ cur.close()
+ conn.close()
+ logger.info("RDS: granted MySQL master privileges for %s", db_id)
+ except Exception as e:
+ logger.warning(
+ "RDS: failed to grant MySQL master privileges for %s: %s",
+ db_id, e)
+
+
+def _wait_for_database_ready(host, port, engine, user, password, db_name, timeout=60):
+ """Block until the database accepts an authenticated connection.
+
+ TCP readiness is not enough for MySQL/Postgres images: they can accept a
+ socket before bootstrap has created users and databases. When a DB driver is
+ unavailable in the lightweight install, fall back to TCP readiness so RDS
+ control-plane behavior remains usable without optional dependencies.
+ """
+ deadline = time.time() + timeout
+ last_error = None
+ while time.time() < deadline:
+ try:
+ if _is_mysql_engine(engine):
+ try:
+ import pymysql
+ except ImportError:
+ return _wait_for_port(host, port, timeout=max(1, deadline - time.time()))
+ conn = pymysql.connect(
+ host=host, port=int(port), user="root",
+ password=password, database=db_name or None,
+ connect_timeout=2, read_timeout=2, write_timeout=2,
+ autocommit=True)
+ elif _is_postgres_engine(engine):
+ try:
+ import psycopg2
+ except ImportError:
+ return _wait_for_port(host, port, timeout=max(1, deadline - time.time()))
+ conn = psycopg2.connect(
+ host=host, port=int(port), user=user,
+ password=password, dbname=db_name or "postgres",
+ connect_timeout=2)
+ else:
+ return _wait_for_port(host, port, timeout=max(1, deadline - time.time()))
+ conn.close()
+ return True
+ except Exception as e:
+ last_error = e
+ time.sleep(0.5)
+ if last_error:
+ logger.debug("RDS: database readiness check failed: %s", last_error)
+ return False
+
+
+def _refresh_cluster_status(cluster_id):
+ if not cluster_id:
+ return
+ cluster = _clusters.get(cluster_id)
+ if not cluster:
+ return
+ if cluster.get("Status") in ("stopped", "deleting"):
+ return
+ member_ids = {
+ m.get("DBInstanceIdentifier")
+ for m in cluster.get("DBClusterMembers", [])
+ if m.get("DBInstanceIdentifier")
+ }
+ if any(
+ inst.get("DBInstanceIdentifier") in member_ids
+ and inst.get("DBInstanceStatus") != "available"
+ for inst in _instances.values()
+ ):
+ cluster["Status"] = "creating"
+ else:
+ cluster["Status"] = "available"
_port_lock = threading.Lock()
@@ -288,6 +393,26 @@ def _resolve_instance(db_id):
return None
+def _sync_cluster_endpoints(cluster):
+ """Point Aurora cluster endpoints at reachable local DB instance endpoints."""
+ members = cluster.get("DBClusterMembers") or []
+ if not members:
+ return
+
+ writer = next((m for m in members if m.get("IsClusterWriter")), members[0])
+ writer_inst = _instances.get(writer.get("DBInstanceIdentifier"))
+ reader_member = next((m for m in members if not m.get("IsClusterWriter")), writer)
+ reader_inst = _instances.get(reader_member.get("DBInstanceIdentifier")) or writer_inst
+
+ if writer_inst and writer_inst.get("Endpoint"):
+ writer_ep = writer_inst["Endpoint"]
+ cluster["Endpoint"] = writer_ep.get("Address", cluster.get("Endpoint", ""))
+ cluster["Port"] = int(writer_ep.get("Port", cluster.get("Port", 0)))
+ if reader_inst and reader_inst.get("Endpoint"):
+ cluster["ReaderEndpoint"] = reader_inst["Endpoint"].get(
+ "Address", cluster.get("ReaderEndpoint", ""))
+
+
def _register_instance_in_cluster(instance):
"""Append instance to parent cluster ``DBClusterMembers`` (Aurora parity)."""
cid = instance.get("DBClusterIdentifier")
@@ -306,6 +431,8 @@ def _register_instance_in_cluster(instance):
"IsClusterWriter": is_writer,
"PromotionTier": int(instance.get("PromotionTier", 1)),
})
+ _sync_cluster_endpoints(cluster)
+ _refresh_cluster_status(cid)
def _unregister_instance_from_clusters(db_id):
@@ -313,6 +440,8 @@ def _unregister_instance_from_clusters(db_id):
for cl in _clusters.values():
mem = cl.get("DBClusterMembers") or []
cl["DBClusterMembers"] = [m for m in mem if m.get("DBInstanceIdentifier") != db_id]
+ _sync_cluster_endpoints(cl)
+ _refresh_cluster_status(cl.get("DBClusterIdentifier"))
# ---------------------------------------------------------------------------
@@ -357,17 +486,20 @@ def _create_db_instance(p):
docker_container_id = None
internal_host = None
internal_port = None
+ real_container_started = False
+ readiness_host = None
+ readiness_port = None
docker_client = _get_docker()
if docker_client:
- host_port = _next_port()
- endpoint_port = host_port
ms_network = _get_ministack_network(docker_client)
image, env, container_port, data_path = _docker_image_for_engine(
engine, engine_version, master_user, master_pass, db_name
)
if image:
try:
+ host_port = _next_port()
+ endpoint_port = host_port
container_kwargs = dict(
image=image, detach=True,
environment=env,
@@ -391,6 +523,7 @@ def _create_db_instance(p):
}
container = docker_client.containers.run(**container_kwargs)
docker_container_id = container.id
+ real_container_started = True
if ms_network:
container.reload()
networks = container.attrs.get(
@@ -402,30 +535,15 @@ def _create_db_instance(p):
internal_port = container_port
endpoint_host = container_ip
endpoint_port = container_port
- def _bg_wait(cip=container_ip, cport=container_port,
- eng=engine, did=db_id, net=ms_network):
- if _wait_for_port(cip, cport):
- logger.info(
- "RDS: %s container for %s ready at "
- "%s:%s (network %s)", eng, did,
- cip, cport, net)
- else:
- logger.warning(
- "RDS: %s container for %s at %s:%s "
- "not ready after timeout", eng,
- did, cip, cport)
- threading.Thread(target=_bg_wait, daemon=True).start()
+ readiness_host = container_ip
+ readiness_port = container_port
else:
logger.info(
"RDS: started %s container for %s on port %s",
engine, db_id, host_port)
else:
- def _bg_wait_port(hp=host_port, eng=engine, did=db_id):
- if _wait_for_port("127.0.0.1", hp):
- logger.info("RDS: %s container for %s ready on port %s", eng, did, hp)
- else:
- logger.warning("RDS: %s container for %s on port %s not ready after timeout", eng, did, hp)
- threading.Thread(target=_bg_wait_port, daemon=True).start()
+ readiness_host = "127.0.0.1"
+ readiness_port = host_port
except Exception as e:
logger.warning("RDS: Docker failed for %s: %s", db_id, e)
@@ -450,7 +568,7 @@ def _create_db_instance(p):
"DBInstanceClass": db_class,
"Engine": engine,
"EngineVersion": engine_version,
- "DBInstanceStatus": "available",
+ "DBInstanceStatus": "creating" if real_container_started else "available",
"MasterUsername": master_user,
"DBName": db_name,
"Endpoint": {
@@ -535,6 +653,33 @@ def _create_db_instance(p):
_instances[db_id] = instance
_register_instance_in_cluster(instance)
+ if real_container_started:
+ ready_host = readiness_host or endpoint_host
+ ready_port = readiness_port or endpoint_port
+ if _wait_for_database_ready(
+ ready_host, ready_port, engine, master_user, master_pass, db_name):
+ if _is_mysql_engine(engine):
+ _grant_mysql_master_user_privileges(
+ ready_host, ready_port, master_user, master_pass, db_id)
+ instance["DBInstanceStatus"] = "available"
+ if cluster_id:
+ cluster = _clusters.get(cluster_id)
+ if cluster:
+ _sync_cluster_endpoints(cluster)
+ _refresh_cluster_status(cluster_id)
+ if ms_network and internal_host:
+ logger.info(
+ "RDS: %s container for %s ready at %s:%s (network %s)",
+ engine, db_id, internal_host, internal_port, ms_network)
+ else:
+ logger.info(
+ "RDS: %s container for %s ready on port %s",
+ engine, db_id, endpoint_port)
+ else:
+ logger.warning(
+ "RDS: %s container for %s at %s:%s not ready after timeout",
+ engine, db_id, ready_host, ready_port)
+
req_tags = _parse_tags(p)
if req_tags:
_tags[arn] = req_tags
diff --git a/ministack/services/rds_data.py b/ministack/services/rds_data.py
index 871d8113..8644f968 100644
--- a/ministack/services/rds_data.py
+++ b/ministack/services/rds_data.py
@@ -32,6 +32,35 @@ def _error(code, message, status=400):
return error_response_json(code, message, status)
+def _transient_database_error(message):
+ return _error("DatabaseUnavailableException", message, 504)
+
+
+def _has_real_endpoint(instance):
+ """Return true when RDS assigned an endpoint backed by a real container."""
+ endpoint = instance.get("Endpoint", {})
+ return (
+ isinstance(endpoint, dict)
+ and bool(endpoint.get("Port"))
+ and bool(instance.get("_docker_container_id"))
+ )
+
+
+def _is_connection_error(exc):
+ err_str = str(exc)
+ return any(
+ marker in err_str
+ for marker in (
+ "Can't connect",
+ "Connection refused",
+ "Connection reset",
+ "timed out",
+ "Name or service not known",
+ "Temporary failure in name resolution",
+ )
+ )
+
+
def _stub_success():
"""Return a minimal successful ExecuteStatement response for mock environments."""
return json_response({
@@ -420,10 +449,10 @@ def _execute_statement(data):
return _error("BadRequestException",
f"Database cluster not found for ARN: {resource_arn}")
- # Check if instance has a real endpoint (Docker container running).
- # Clusters have Endpoint as a string (hostname); instances have it as a dict.
- endpoint = instance.get("Endpoint", {})
- if isinstance(endpoint, str) or not endpoint.get("Port"):
+ # Keep stub mode for intentional mock environments only. Once RDS has a
+ # real container-backed endpoint, connection failures must surface as
+ # transient errors instead of acknowledging writes that never reached MySQL.
+ if not _has_real_endpoint(instance):
logger.info("No endpoint for %s, using stub mode", resource_arn)
return _stub_execute(resource_arn, sql)
@@ -478,21 +507,13 @@ def _execute_statement(data):
except ImportError as e:
if own_conn and conn:
conn.close()
- if not getattr(_execute_statement, "_import_warned", False):
- logger.warning("DB driver not available, using stub: %s", e)
- _execute_statement._import_warned = True
- return _stub_execute(resource_arn, sql)
+ return _error("BadRequestException", str(e))
except Exception as e:
if own_conn and conn:
conn.close()
- # Connection errors (e.g. when RDS containers are not reachable from
- # within the MiniStack container) should fall back to stubs rather than
- # failing the caller. This covers LAMBDA_EXECUTOR=local where the
- # MySQL sidecar container is not network-accessible.
- err_str = str(e)
- if "Can't connect" in err_str or "Connection refused" in err_str:
- logger.warning("DB connection failed, using stub: %s", e)
- return _stub_execute(resource_arn, sql)
+ if _is_connection_error(e):
+ logger.warning("DB connection failed for real endpoint: %s", e)
+ return _transient_database_error(f"Database endpoint is not available: {e}")
return _error("BadRequestException", f"Database error: {e}")
diff --git a/tests/test_rds.py b/tests/test_rds.py
index ef4f2012..cb55c54f 100644
--- a/tests/test_rds.py
+++ b/tests/test_rds.py
@@ -899,6 +899,90 @@ def test_rds_aurora_cluster_lists_instance_member(rds):
assert any(m["DBInstanceIdentifier"] == iid for m in members)
+def test_rds_aurora_cluster_endpoints_follow_backing_instance(rds):
+ """Aurora cluster endpoints should be reachable through the local instance."""
+ cid = f"epclus-{_uuid_mod.uuid4().hex[:10]}"
+ iid = f"{cid}-writer"
+ rds.create_db_cluster(
+ DBClusterIdentifier=cid,
+ Engine="aurora-mysql",
+ MasterUsername="admin",
+ MasterUserPassword="pw",
+ )
+ rds.create_db_instance(
+ DBInstanceIdentifier=iid,
+ DBClusterIdentifier=cid,
+ DBInstanceClass="db.r6g.large",
+ Engine="aurora-mysql",
+ )
+
+ inst = rds.describe_db_instances(DBInstanceIdentifier=iid)["DBInstances"][0]
+ cluster = rds.describe_db_clusters(DBClusterIdentifier=cid)["DBClusters"][0]
+
+ assert cluster["Endpoint"] == inst["Endpoint"]["Address"]
+ assert cluster["ReaderEndpoint"] == inst["Endpoint"]["Address"]
+ assert cluster["Port"] == inst["Endpoint"]["Port"]
+
+
+def test_rds_mysql_master_user_privilege_grants(monkeypatch):
+ """MySQL master users get admin grants, with dynamic grants best-effort."""
+ import sys
+ import types
+
+ from ministack.services import rds as m
+
+ calls = []
+
+ class FakeCursor:
+ def execute(self, sql, params=None):
+ calls.append((sql, params))
+ if "APPLICATION_PASSWORD_ADMIN" in sql:
+ raise Exception("unsupported privilege")
+
+ def close(self):
+ calls.append(("cursor.close", None))
+
+ class FakeConnection:
+ def cursor(self):
+ return FakeCursor()
+
+ def close(self):
+ calls.append(("connection.close", None))
+
+ def fake_connect(**kwargs):
+ calls.append(("connect", kwargs))
+ return FakeConnection()
+
+ monkeypatch.setitem(
+ sys.modules,
+ "pymysql",
+ types.SimpleNamespace(connect=fake_connect),
+ )
+
+ m._grant_mysql_master_user_privileges(
+ "10.0.0.12", 3306, "admin", "password123", "mysql-test")
+
+ assert calls[0] == (
+ "connect",
+ {
+ "host": "10.0.0.12",
+ "port": 3306,
+ "user": "root",
+ "password": "password123",
+ "autocommit": True,
+ },
+ )
+ assert (
+ "CREATE USER IF NOT EXISTS %s@'%%' IDENTIFIED BY %s",
+ ("admin", "password123"),
+ ) in calls
+ assert (
+ "GRANT ALL PRIVILEGES ON *.* TO %s@'%%' WITH GRANT OPTION",
+ ("admin",),
+ ) in calls
+ assert ("FLUSH PRIVILEGES", None) in calls
+
+
def test_rds_modify_cluster_password(rds):
"""ModifyDBCluster with MasterUserPassword succeeds."""
rds.create_db_cluster(
diff --git a/tests/test_rds_data.py b/tests/test_rds_data.py
index c8794d2a..9b250cb3 100644
--- a/tests/test_rds_data.py
+++ b/tests/test_rds_data.py
@@ -364,6 +364,140 @@ def test_rds_data_stub_drop_user(rds, sm):
assert "tempuser" not in names
+def test_rds_data_real_endpoint_connection_failure_is_transient(monkeypatch):
+ """Endpoint-backed clusters must not acknowledge writes through stub mode."""
+ from ministack.services import rds_data
+
+ rds_data.reset()
+ resource_arn = f"arn:aws:rds:{REGION}:{ACCOUNT_ID}:cluster:real-cluster"
+ instance = {
+ "Endpoint": {"Address": "10.0.0.10", "Port": 3306},
+ "_docker_container_id": "container-id",
+ "Engine": "aurora-mysql",
+ }
+
+ monkeypatch.setattr(rds_data, "_resolve_cluster", lambda _arn: (instance, "aurora-mysql"))
+ monkeypatch.setattr(rds_data, "_get_secret_credentials", lambda _arn: ("admin", "pw"))
+
+ def _raise_connection_error(*_args, **_kwargs):
+ raise OSError("Connection refused")
+
+ monkeypatch.setattr(rds_data, "_connect", _raise_connection_error)
+
+ status, headers, body = rds_data._execute_statement({
+ "resourceArn": resource_arn,
+ "secretArn": FAKE_SECRET_ARN,
+ "sql": "CREATE USER 'lost_write'@'%' IDENTIFIED BY 'pw'",
+ })
+ payload = json.loads(body)
+
+ assert status == 504
+ assert headers["x-amzn-errortype"] == "DatabaseUnavailableException"
+ assert payload["__type"] == "DatabaseUnavailableException"
+ assert "lost_write" not in rds_data._stub_users.get("real-cluster", set())
+
+
+def test_rds_data_non_container_endpoint_keeps_stub_mode(monkeypatch):
+ """Control-plane-only instances still use the lightweight SQL stub."""
+ from ministack.services import rds_data
+
+ rds_data.reset()
+ resource_arn = f"arn:aws:rds:{REGION}:{ACCOUNT_ID}:cluster:stub-cluster"
+ instance = {
+ "Endpoint": {"Address": "localhost", "Port": 3306},
+ "_docker_container_id": None,
+ "Engine": "aurora-mysql",
+ }
+ monkeypatch.setattr(rds_data, "_resolve_cluster", lambda _arn: (instance, "aurora-mysql"))
+
+ status, _headers, body = rds_data._execute_statement({
+ "resourceArn": resource_arn,
+ "secretArn": FAKE_SECRET_ARN,
+ "sql": "CREATE USER 'stub_user'@'%' IDENTIFIED BY 'pw'",
+ })
+ payload = json.loads(body)
+
+ assert status == 200
+ assert payload["records"] == []
+ assert "stub_user" in rds_data._stub_users.get("stub-cluster", set())
+
+
+def test_rds_data_lock_wait_timeout_is_not_connection_error():
+ """MySQL lock wait timeout is a SQL error, not endpoint unavailability."""
+ from ministack.services import rds_data
+
+ assert not rds_data._is_connection_error(
+ Exception("(1205, 'Lock wait timeout exceeded; try restarting transaction')")
+ )
+
+
+def test_rds_cluster_status_tracks_member_readiness():
+ """Parent clusters remain creating until their member instance is available."""
+ from ministack.services import rds
+
+ cluster_id = "readiness-cluster"
+ instance_id = "readiness-instance"
+ rds._clusters[cluster_id] = {
+ "DBClusterIdentifier": cluster_id,
+ "Status": "available",
+ "DBClusterMembers": [],
+ }
+ rds._instances[instance_id] = {
+ "DBInstanceIdentifier": instance_id,
+ "DBClusterIdentifier": cluster_id,
+ "DBInstanceStatus": "creating",
+ "PromotionTier": 1,
+ }
+
+ try:
+ rds._register_instance_in_cluster(rds._instances[instance_id])
+ assert rds._clusters[cluster_id]["Status"] == "creating"
+
+ rds._instances[instance_id]["DBInstanceStatus"] = "available"
+ rds._refresh_cluster_status(cluster_id)
+ assert rds._clusters[cluster_id]["Status"] == "available"
+ finally:
+ rds._instances.pop(instance_id, None)
+ rds._clusters.pop(cluster_id, None)
+
+
+def test_rds_cluster_endpoints_sync_to_backing_instance():
+ """Aurora cluster endpoints track the registered local writer instance."""
+ from ministack.services import rds
+
+ cluster_id = "endpoint-sync-cluster"
+ instance_id = "endpoint-sync-instance"
+ rds._clusters[cluster_id] = {
+ "DBClusterIdentifier": cluster_id,
+ "Status": "available",
+ "Endpoint": "original.cluster.local",
+ "ReaderEndpoint": "original-ro.cluster.local",
+ "Port": 3306,
+ "DBClusterMembers": [],
+ }
+ rds._instances[instance_id] = {
+ "DBInstanceIdentifier": instance_id,
+ "DBClusterIdentifier": cluster_id,
+ "DBInstanceStatus": "available",
+ "Endpoint": {
+ "Address": "10.0.0.12",
+ "Port": 3306,
+ },
+ "PromotionTier": 1,
+ }
+
+ try:
+ rds._register_instance_in_cluster(rds._instances[instance_id])
+
+ cluster = rds._clusters[cluster_id]
+ assert cluster["Endpoint"] == "10.0.0.12"
+ assert cluster["ReaderEndpoint"] == "10.0.0.12"
+ assert cluster["Port"] == 3306
+ finally:
+ rds._instances.pop(instance_id, None)
+ rds._clusters.pop(cluster_id, None)
+
+
def test_rds_data_secret_credentials_parsing():
"""_get_secret_credentials extracts username and password from secret."""
from ministack.core.responses import set_request_account_id