Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
Fix RDS Data API readiness race (#666)
Jay Janssena799e48affff
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex b876c135..266216e0 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -33,6 +33,10 @@ Versioning follows [Semantic Versioning](https://semver.org/).### Changed- **Test harness — xdist session-startup reset coordination** — the per-worker `autouse` `reset_server` fixture previously had every xdist worker hit `/_ministack/reset` on session start, so a slower worker's reset could fire after a faster worker had already begun creating fixtures, wiping that state mid-test (most visibly as occasional `list_functions()` empty results in `test_lambda_create_invoke`). The first worker now wins an `O_EXCL` file lock and runs the reset; the others wait briefly for a marker file and skip. Single-process pytest (no xdist) keeps the original behaviour. Removes a class of CI flakes that only reproduced under parallel load.+### Fixed+- **RDS Data API no longer acknowledges writes through the SQL stub for real containers that are still booting** — Docker-backed RDS instances now wait for an authenticated database connection before reporting `available`, and their parent clusters remain `creating` while member instances are not ready. If an endpoint-backed container still cannot be reached, RDS Data API returns `DatabaseUnavailableException` instead of silently tracking `CREATE USER` / `GRANT` statements in memory. Stub SQL mode remains available for control-plane-only clusters without a real backing container. Contributed by @jayjanssen.+- **RDS Aurora MySQL local endpoint and master-user parity** — Aurora cluster endpoints now track the reachable backing DB instance endpoint after cluster members are created, so Lambda containers using `DescribeDBClusters.Endpoint` can connect to MiniStack-backed databases. MySQL and Aurora MySQL containers also grant the configured master username AWS/RDS-like global privileges after the server is reachable, with version-specific dynamic privileges treated as best-effort. Contributed by @jayjanssen.+---## [1.3.41] — 2026-05-16diff --git a/ministack/services/rds.py b/ministack/services/rds.pyindex c7190e00..41561bd9 100644--- a/ministack/services/rds.py+++ b/ministack/services/rds.py@@ -180,6 +180,111 @@ def _wait_for_port(host, port, timeout=60):return False+def _is_mysql_engine(engine):+ return any(e in engine for e in ("mysql", "aurora-mysql", "mariadb"))+++def _is_postgres_engine(engine):+ return any(e in engine for e in ("postgres", "aurora-postgresql"))+++def _grant_mysql_master_user_privileges(host, port, master_user, master_pass, db_id):+ """Grant the emulated MySQL master user AWS/RDS-like admin privileges."""+ try:+ import pymysql+ conn = pymysql.connect(+ host=host, port=int(port), user="root",+ password=master_pass, autocommit=True)+ cur = conn.cursor()+ cur.execute(+ "CREATE USER IF NOT EXISTS %s@'%%' IDENTIFIED BY %s",+ (master_user, master_pass),+ )+ cur.execute(+ "GRANT ALL PRIVILEGES ON *.* TO %s@'%%' WITH GRANT OPTION",+ (master_user,),+ )+ for privilege in ("APPLICATION_PASSWORD_ADMIN",):+ try:+ cur.execute(f"GRANT {privilege} ON *.* TO %s@'%%'", (master_user,))+ except Exception as e:+ logger.debug(+ "RDS: MySQL privilege %s unsupported for %s: %s",+ privilege, db_id, e)+ cur.execute("FLUSH PRIVILEGES")+ cur.close()+ conn.close()+ logger.info("RDS: granted MySQL master privileges for %s", db_id)+ except Exception as e:+ logger.warning(+ "RDS: failed to grant MySQL master privileges for %s: %s",+ db_id, e)+++def _wait_for_database_ready(host, port, engine, user, password, db_name, timeout=60):+ """Block until the database accepts an authenticated connection.++ TCP readiness is not enough for MySQL/Postgres images: they can accept a+ socket before bootstrap has created users and databases. When a DB driver is+ unavailable in the lightweight install, fall back to TCP readiness so RDS+ control-plane behavior remains usable without optional dependencies.+ """+ deadline = time.time() + timeout+ last_error = None+ while time.time() < deadline:+ try:+ if _is_mysql_engine(engine):+ try:+ import pymysql+ except ImportError:+ return _wait_for_port(host, port, timeout=max(1, deadline - time.time()))+ conn = pymysql.connect(+ host=host, port=int(port), user="root",+ password=password, database=db_name or None,+ connect_timeout=2, read_timeout=2, write_timeout=2,+ autocommit=True)+ elif _is_postgres_engine(engine):+ try:+ import psycopg2+ except ImportError:+ return _wait_for_port(host, port, timeout=max(1, deadline - time.time()))+ conn = psycopg2.connect(+ host=host, port=int(port), user=user,+ password=password, dbname=db_name or "postgres",+ connect_timeout=2)+ else:+ return _wait_for_port(host, port, timeout=max(1, deadline - time.time()))+ conn.close()+ return True+ except Exception as e:+ last_error = e+ time.sleep(0.5)+ if last_error:+ logger.debug("RDS: database readiness check failed: %s", last_error)+ return False+++def _refresh_cluster_status(cluster_id):+ if not cluster_id:+ return+ cluster = _clusters.get(cluster_id)+ if not cluster:+ return+ if cluster.get("Status") in ("stopped", "deleting"):+ return+ member_ids = {+ m.get("DBInstanceIdentifier")+ for m in cluster.get("DBClusterMembers", [])+ if m.get("DBInstanceIdentifier")+ }+ if any(+ inst.get("DBInstanceIdentifier") in member_ids+ and inst.get("DBInstanceStatus") != "available"+ for inst in _instances.values()+ ):+ cluster["Status"] = "creating"+ else:+ cluster["Status"] = "available"_port_lock = threading.Lock()@@ -288,6 +393,26 @@ def _resolve_instance(db_id):return None+def _sync_cluster_endpoints(cluster):+ """Point Aurora cluster endpoints at reachable local DB instance endpoints."""+ members = cluster.get("DBClusterMembers") or []+ if not members:+ return++ writer = next((m for m in members if m.get("IsClusterWriter")), members[0])+ writer_inst = _instances.get(writer.get("DBInstanceIdentifier"))+ reader_member = next((m for m in members if not m.get("IsClusterWriter")), writer)+ reader_inst = _instances.get(reader_member.get("DBInstanceIdentifier")) or writer_inst++ if writer_inst and writer_inst.get("Endpoint"):+ writer_ep = writer_inst["Endpoint"]+ cluster["Endpoint"] = writer_ep.get("Address", cluster.get("Endpoint", ""))+ cluster["Port"] = int(writer_ep.get("Port", cluster.get("Port", 0)))+ if reader_inst and reader_inst.get("Endpoint"):+ cluster["ReaderEndpoint"] = reader_inst["Endpoint"].get(+ "Address", cluster.get("ReaderEndpoint", ""))++def _register_instance_in_cluster(instance):"""Append instance to parent cluster ``DBClusterMembers`` (Aurora parity)."""cid = instance.get("DBClusterIdentifier")@@ -306,6 +431,8 @@ def _register_instance_in_cluster(instance):"IsClusterWriter": is_writer,"PromotionTier": int(instance.get("PromotionTier", 1)),})+ _sync_cluster_endpoints(cluster)+ _refresh_cluster_status(cid)def _unregister_instance_from_clusters(db_id):@@ -313,6 +440,8 @@ def _unregister_instance_from_clusters(db_id):for cl in _clusters.values():mem = cl.get("DBClusterMembers") or []cl["DBClusterMembers"] = [m for m in mem if m.get("DBInstanceIdentifier") != db_id]+ _sync_cluster_endpoints(cl)+ _refresh_cluster_status(cl.get("DBClusterIdentifier"))# ---------------------------------------------------------------------------@@ -357,17 +486,20 @@ def _create_db_instance(p):docker_container_id = Noneinternal_host = Noneinternal_port = None+ real_container_started = False+ readiness_host = None+ readiness_port = Nonedocker_client = _get_docker()if docker_client:- host_port = _next_port()- endpoint_port = host_portms_network = _get_ministack_network(docker_client)image, env, container_port, data_path = _docker_image_for_engine(engine, engine_version, master_user, master_pass, db_name)if image:try:+ host_port = _next_port()+ endpoint_port = host_portcontainer_kwargs = dict(image=image, detach=True,environment=env,@@ -391,6 +523,7 @@ def _create_db_instance(p):}container = docker_client.containers.run(**container_kwargs)docker_container_id = container.id+ real_container_started = Trueif ms_network:container.reload()networks = container.attrs.get(@@ -402,30 +535,15 @@ def _create_db_instance(p):internal_port = container_portendpoint_host = container_ipendpoint_port = container_port- def _bg_wait(cip=container_ip, cport=container_port,- eng=engine, did=db_id, net=ms_network):- if _wait_for_port(cip, cport):- logger.info(- "RDS: %s container for %s ready at "- "%s:%s (network %s)", eng, did,- cip, cport, net)- else:- logger.warning(- "RDS: %s container for %s at %s:%s "- "not ready after timeout", eng,- did, cip, cport)- threading.Thread(target=_bg_wait, daemon=True).start()+ readiness_host = container_ip+ readiness_port = container_portelse:logger.info("RDS: started %s container for %s on port %s",engine, db_id, host_port)else:- def _bg_wait_port(hp=host_port, eng=engine, did=db_id):- if _wait_for_port("127.0.0.1", hp):- logger.info("RDS: %s container for %s ready on port %s", eng, did, hp)- else:- logger.warning("RDS: %s container for %s on port %s not ready after timeout", eng, did, hp)- threading.Thread(target=_bg_wait_port, daemon=True).start()+ readiness_host = "127.0.0.1"+ readiness_port = host_portexcept Exception as e:logger.warning("RDS: Docker failed for %s: %s", db_id, e)@@ -450,7 +568,7 @@ def _create_db_instance(p):"DBInstanceClass": db_class,"Engine": engine,"EngineVersion": engine_version,- "DBInstanceStatus": "available",+ "DBInstanceStatus": "creating" if real_container_started else "available","MasterUsername": master_user,"DBName": db_name,"Endpoint": {@@ -535,6 +653,33 @@ def _create_db_instance(p):_instances[db_id] = instance_register_instance_in_cluster(instance)+ if real_container_started:+ ready_host = readiness_host or endpoint_host+ ready_port = readiness_port or endpoint_port+ if _wait_for_database_ready(+ ready_host, ready_port, engine, master_user, master_pass, db_name):+ if _is_mysql_engine(engine):+ _grant_mysql_master_user_privileges(+ ready_host, ready_port, master_user, master_pass, db_id)+ instance["DBInstanceStatus"] = "available"+ if cluster_id:+ cluster = _clusters.get(cluster_id)+ if cluster:+ _sync_cluster_endpoints(cluster)+ _refresh_cluster_status(cluster_id)+ if ms_network and internal_host:+ logger.info(+ "RDS: %s container for %s ready at %s:%s (network %s)",+ engine, db_id, internal_host, internal_port, ms_network)+ else:+ logger.info(+ "RDS: %s container for %s ready on port %s",+ engine, db_id, endpoint_port)+ else:+ logger.warning(+ "RDS: %s container for %s at %s:%s not ready after timeout",+ engine, db_id, ready_host, ready_port)+req_tags = _parse_tags(p)if req_tags:_tags[arn] = req_tagsdiff --git a/ministack/services/rds_data.py b/ministack/services/rds_data.pyindex 871d8113..8644f968 100644--- a/ministack/services/rds_data.py+++ b/ministack/services/rds_data.py@@ -32,6 +32,35 @@ def _error(code, message, status=400):return error_response_json(code, message, status)+def _transient_database_error(message):+ return _error("DatabaseUnavailableException", message, 504)+++def _has_real_endpoint(instance):+ """Return true when RDS assigned an endpoint backed by a real container."""+ endpoint = instance.get("Endpoint", {})+ return (+ isinstance(endpoint, dict)+ and bool(endpoint.get("Port"))+ and bool(instance.get("_docker_container_id"))+ )+++def _is_connection_error(exc):+ err_str = str(exc)+ return any(+ marker in err_str+ for marker in (+ "Can't connect",+ "Connection refused",+ "Connection reset",+ "timed out",+ "Name or service not known",+ "Temporary failure in name resolution",+ )+ )++def _stub_success():"""Return a minimal successful ExecuteStatement response for mock environments."""return json_response({@@ -420,10 +449,10 @@ def _execute_statement(data):return _error("BadRequestException",f"Database cluster not found for ARN: {resource_arn}")- # Check if instance has a real endpoint (Docker container running).- # Clusters have Endpoint as a string (hostname); instances have it as a dict.- endpoint = instance.get("Endpoint", {})- if isinstance(endpoint, str) or not endpoint.get("Port"):+ # Keep stub mode for intentional mock environments only. Once RDS has a+ # real container-backed endpoint, connection failures must surface as+ # transient errors instead of acknowledging writes that never reached MySQL.+ if not _has_real_endpoint(instance):logger.info("No endpoint for %s, using stub mode", resource_arn)return _stub_execute(resource_arn, sql)@@ -478,21 +507,13 @@ def _execute_statement(data):except ImportError as e:if own_conn and conn:conn.close()- if not getattr(_execute_statement, "_import_warned", False):- logger.warning("DB driver not available, using stub: %s", e)- _execute_statement._import_warned = True- return _stub_execute(resource_arn, sql)+ return _error("BadRequestException", str(e))except Exception as e:if own_conn and conn:conn.close()- # Connection errors (e.g. when RDS containers are not reachable from- # within the MiniStack container) should fall back to stubs rather than- # failing the caller. This covers LAMBDA_EXECUTOR=local where the- # MySQL sidecar container is not network-accessible.- err_str = str(e)- if "Can't connect" in err_str or "Connection refused" in err_str:- logger.warning("DB connection failed, using stub: %s", e)- return _stub_execute(resource_arn, sql)+ if _is_connection_error(e):+ logger.warning("DB connection failed for real endpoint: %s", e)+ return _transient_database_error(f"Database endpoint is not available: {e}")return _error("BadRequestException", f"Database error: {e}")diff --git a/tests/test_rds.py b/tests/test_rds.pyindex ef4f2012..cb55c54f 100644--- a/tests/test_rds.py+++ b/tests/test_rds.py@@ -899,6 +899,90 @@ def test_rds_aurora_cluster_lists_instance_member(rds):assert any(m["DBInstanceIdentifier"] == iid for m in members)+def test_rds_aurora_cluster_endpoints_follow_backing_instance(rds):+ """Aurora cluster endpoints should be reachable through the local instance."""+ cid = f"epclus-{_uuid_mod.uuid4().hex[:10]}"+ iid = f"{cid}-writer"+ rds.create_db_cluster(+ DBClusterIdentifier=cid,+ Engine="aurora-mysql",+ MasterUsername="admin",+ MasterUserPassword="pw",+ )+ rds.create_db_instance(+ DBInstanceIdentifier=iid,+ DBClusterIdentifier=cid,+ DBInstanceClass="db.r6g.large",+ Engine="aurora-mysql",+ )++ inst = rds.describe_db_instances(DBInstanceIdentifier=iid)["DBInstances"][0]+ cluster = rds.describe_db_clusters(DBClusterIdentifier=cid)["DBClusters"][0]++ assert cluster["Endpoint"] == inst["Endpoint"]["Address"]+ assert cluster["ReaderEndpoint"] == inst["Endpoint"]["Address"]+ assert cluster["Port"] == inst["Endpoint"]["Port"]+++def test_rds_mysql_master_user_privilege_grants(monkeypatch):+ """MySQL master users get admin grants, with dynamic grants best-effort."""+ import sys+ import types++ from ministack.services import rds as m++ calls = []++ class FakeCursor:+ def execute(self, sql, params=None):+ calls.append((sql, params))+ if "APPLICATION_PASSWORD_ADMIN" in sql:+ raise Exception("unsupported privilege")++ def close(self):+ calls.append(("cursor.close", None))++ class FakeConnection:+ def cursor(self):+ return FakeCursor()++ def close(self):+ calls.append(("connection.close", None))++ def fake_connect(**kwargs):+ calls.append(("connect", kwargs))+ return FakeConnection()++ monkeypatch.setitem(+ sys.modules,+ "pymysql",+ types.SimpleNamespace(connect=fake_connect),+ )++ m._grant_mysql_master_user_privileges(+ "10.0.0.12", 3306, "admin", "password123", "mysql-test")++ assert calls[0] == (+ "connect",+ {+ "host": "10.0.0.12",+ "port": 3306,+ "user": "root",+ "password": "password123",+ "autocommit": True,+ },+ )+ assert (+ "CREATE USER IF NOT EXISTS %s@'%%' IDENTIFIED BY %s",+ ("admin", "password123"),+ ) in calls+ assert (+ "GRANT ALL PRIVILEGES ON *.* TO %s@'%%' WITH GRANT OPTION",+ ("admin",),+ ) in calls+ assert ("FLUSH PRIVILEGES", None) in calls++def test_rds_modify_cluster_password(rds):"""ModifyDBCluster with MasterUserPassword succeeds."""rds.create_db_cluster(diff --git a/tests/test_rds_data.py b/tests/test_rds_data.pyindex c8794d2a..9b250cb3 100644--- a/tests/test_rds_data.py+++ b/tests/test_rds_data.py@@ -364,6 +364,140 @@ def test_rds_data_stub_drop_user(rds, sm):assert "tempuser" not in names+def test_rds_data_real_endpoint_connection_failure_is_transient(monkeypatch):+ """Endpoint-backed clusters must not acknowledge writes through stub mode."""+ from ministack.services import rds_data++ rds_data.reset()+ resource_arn = f"arn:aws:rds:{REGION}:{ACCOUNT_ID}:cluster:real-cluster"+ instance = {+ "Endpoint": {"Address": "10.0.0.10", "Port": 3306},+ "_docker_container_id": "container-id",+ "Engine": "aurora-mysql",+ }++ monkeypatch.setattr(rds_data, "_resolve_cluster", lambda _arn: (instance, "aurora-mysql"))+ monkeypatch.setattr(rds_data, "_get_secret_credentials", lambda _arn: ("admin", "pw"))++ def _raise_connection_error(*_args, **_kwargs):+ raise OSError("Connection refused")++ monkeypatch.setattr(rds_data, "_connect", _raise_connection_error)++ status, headers, body = rds_data._execute_statement({+ "resourceArn": resource_arn,+ "secretArn": FAKE_SECRET_ARN,+ "sql": "CREATE USER 'lost_write'@'%' IDENTIFIED BY 'pw'",+ })+ payload = json.loads(body)++ assert status == 504+ assert headers["x-amzn-errortype"] == "DatabaseUnavailableException"+ assert payload["__type"] == "DatabaseUnavailableException"+ assert "lost_write" not in rds_data._stub_users.get("real-cluster", set())+++def test_rds_data_non_container_endpoint_keeps_stub_mode(monkeypatch):+ """Control-plane-only instances still use the lightweight SQL stub."""+ from ministack.services import rds_data++ rds_data.reset()+ resource_arn = f"arn:aws:rds:{REGION}:{ACCOUNT_ID}:cluster:stub-cluster"+ instance = {+ "Endpoint": {"Address": "localhost", "Port": 3306},+ "_docker_container_id": None,+ "Engine": "aurora-mysql",+ }+ monkeypatch.setattr(rds_data, "_resolve_cluster", lambda _arn: (instance, "aurora-mysql"))++ status, _headers, body = rds_data._execute_statement({+ "resourceArn": resource_arn,+ "secretArn": FAKE_SECRET_ARN,+ "sql": "CREATE USER 'stub_user'@'%' IDENTIFIED BY 'pw'",+ })+ payload = json.loads(body)++ assert status == 200+ assert payload["records"] == []+ assert "stub_user" in rds_data._stub_users.get("stub-cluster", set())+++def test_rds_data_lock_wait_timeout_is_not_connection_error():+ """MySQL lock wait timeout is a SQL error, not endpoint unavailability."""+ from ministack.services import rds_data++ assert not rds_data._is_connection_error(+ Exception("(1205, 'Lock wait timeout exceeded; try restarting transaction')")+ )+++def test_rds_cluster_status_tracks_member_readiness():+ """Parent clusters remain creating until their member instance is available."""+ from ministack.services import rds++ cluster_id = "readiness-cluster"+ instance_id = "readiness-instance"+ rds._clusters[cluster_id] = {+ "DBClusterIdentifier": cluster_id,+ "Status": "available",+ "DBClusterMembers": [],+ }+ rds._instances[instance_id] = {+ "DBInstanceIdentifier": instance_id,+ "DBClusterIdentifier": cluster_id,+ "DBInstanceStatus": "creating",+ "PromotionTier": 1,+ }++ try:+ rds._register_instance_in_cluster(rds._instances[instance_id])+ assert rds._clusters[cluster_id]["Status"] == "creating"++ rds._instances[instance_id]["DBInstanceStatus"] = "available"+ rds._refresh_cluster_status(cluster_id)+ assert rds._clusters[cluster_id]["Status"] == "available"+ finally:+ rds._instances.pop(instance_id, None)+ rds._clusters.pop(cluster_id, None)+++def test_rds_cluster_endpoints_sync_to_backing_instance():+ """Aurora cluster endpoints track the registered local writer instance."""+ from ministack.services import rds++ cluster_id = "endpoint-sync-cluster"+ instance_id = "endpoint-sync-instance"+ rds._clusters[cluster_id] = {+ "DBClusterIdentifier": cluster_id,+ "Status": "available",+ "Endpoint": "original.cluster.local",+ "ReaderEndpoint": "original-ro.cluster.local",+ "Port": 3306,+ "DBClusterMembers": [],+ }+ rds._instances[instance_id] = {+ "DBInstanceIdentifier": instance_id,+ "DBClusterIdentifier": cluster_id,+ "DBInstanceStatus": "available",+ "Endpoint": {+ "Address": "10.0.0.12",+ "Port": 3306,+ },+ "PromotionTier": 1,+ }++ try:+ rds._register_instance_in_cluster(rds._instances[instance_id])++ cluster = rds._clusters[cluster_id]+ assert cluster["Endpoint"] == "10.0.0.12"+ assert cluster["ReaderEndpoint"] == "10.0.0.12"+ assert cluster["Port"] == 3306+ finally:+ rds._instances.pop(instance_id, None)+ rds._clusters.pop(cluster_id, None)++def test_rds_data_secret_credentials_parsing():"""_get_secret_credentials extracts username and password from secret."""from ministack.core.responses import set_request_account_id