Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
rds: enforce global-database Stop restriction and re-link replication on Start (#1438)
kiran01bmc3f09b739481
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9b3be612..037c01ad 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,8 @@ Versioning follows [Semantic Versioning](https://semver.org/).## [Unreleased]+### Fixed+- **RDS — global Aurora stop/start preserves topology and MySQL replication** — `StopDBCluster` and `StartDBCluster` allowed individual members of a multi-cluster Aurora global database to transition independently, and deleting a primary's last instance stopped shared compute that secondaries still needed. Starts and stops are now limited to sole-member global databases (`InvalidDBClusterStateFault`, 400), last-instance deletion preserves compute while other global members exist, and a successful recreate resets stale MySQL replica state before reconfiguring or deferring replication after readiness without overwriting a deliberate legacy-volume failure. Contributed by @kiran01bm.### Added- **IoT Core — native mTLS MQTT listener on port 8883** — the embedded broker spoke MQTT only over WebSocket, so AWS IoT Device SDK binaries could not connect. The broker now also listens for MQTT over TLS on port 8883 (`IOT_MTLS_ENABLED=0` turns it off, `IOT_MTLS_PORT` moves it), on by default when `cryptography` is available, configured exactly like the Transfer Family SFTP listener. The broker certificate is minted from the local CA (`GET /_ministack/iot/ca.pem`). The client certificate is optional and follows the rule S3 applies to a presigned URL: a client that presents none is served under `MINISTACK_ACCOUNT_ID`, exactly like an unsigned MQTT-over-WebSocket upgrade, while one that presents a certificate has it read against the registry. Registered `ACTIVE` in exactly one account serves that account and region; unknown, not `ACTIVE`, or `ACTIVE` in several accounts is refused with a "not authorized" CONNACK (0x05, reason code 0x87 for MQTT 5 clients), so `UpdateCertificate` to `INACTIVE` cuts a device off the way it does on AWS. Contributed by @iot-rocket.diff --git a/ministack/services/rds.py b/ministack/services/rds.pyindex 07ee791c..81cd6e4d 100644--- a/ministack/services/rds.py+++ b/ministack/services/rds.py@@ -3679,12 +3679,17 @@ def _delete_db_instance(p):if shared_cluster_id:cluster = _resolve_cluster_in_request_region(shared_cluster_id)if cluster and not cluster.get("DBClusterMembers"):- if _mysql_replication_secondary(cluster):+ global_cluster, _member = _global_cluster_member_for_cluster(cluster)+ if (+ global_cluster+ and len(global_cluster.get("GlobalClusterMembers", [])) > 1+ ):# Aurora global headless secondaries have no query compute but- # their storage continues to synchronize. MiniStack's single- # shared process is also the applier, so preserve it until the- # secondary is detached, deleted, or gains compute again.- cluster["_mysql_headless_applier_required"] = True+ # their storage continues to synchronize. MiniStack's shared+ # process also sustains the global topology, so preserve it+ # for both writers and secondaries while other members exist.+ if _mysql_replication_secondary(cluster):+ cluster["_mysql_headless_applier_required"] = Trueelse:_stop_cluster_shared_container(shared_cluster_id, cluster)@@ -5226,6 +5231,19 @@ def _start_db_cluster(p):"stopped,inaccessible-encryption-credentials-recoverable.",400,)+ global_cluster, _member = _global_cluster_member_for_cluster(cluster)+ if (+ global_cluster+ and len(global_cluster.get("GlobalClusterMembers", [])) > 1+ ):+ # Message text is verbatim from the AWS Aurora global-database+ # limitations documentation, not from a captured wire transcript.+ return _error(+ "InvalidDBClusterStateFault",+ "You can only stop and start a cluster that's part of an Aurora "+ "global database if it's the only cluster in the global database.",+ 400,+ )member_instances = _cluster_member_instances(cluster)docker_client = _get_docker()@@ -5242,6 +5260,7 @@ def _start_db_cluster(p):return _xml(200, "StartDBClusterResponse",f"<StartDBClusterResult><DBCluster>{_cluster_xml(cluster)}</DBCluster></StartDBClusterResult>")+ compute_recreated = Falseif had_compute:start_result = _restart_cluster_shared_container(cluster_id, cluster)if start_result.get("failed"):@@ -5251,6 +5270,7 @@ def _start_db_cluster(p):start_result = _start_cluster_shared_container(cluster_id, cluster, remove_stale=True,)+ compute_recreated = start_result.get("started", False)else:# Initialized storage without a container id happens after a host# restart: persistence drops the container id but the stable-named@@ -5258,6 +5278,7 @@ def _start_db_cluster(p):start_result = _start_cluster_shared_container(cluster_id, cluster, remove_stale=True,)+ compute_recreated = start_result.get("started", False)if not start_result.get("started"):if start_result.get("failed"):@@ -5301,6 +5322,7 @@ def _start_db_cluster(p):container_epoch=start_result.get("container_epoch"),ready_host=start_result.get("readiness_host"),ready_port=start_result.get("readiness_port"),+ compute_recreated=compute_recreated,):cluster = _clusters.get(cluster_id)if not cluster:@@ -5416,6 +5438,18 @@ def _start_db_cluster(p):cluster_id,)cluster["_shared_container_ready"] = True+ if _aurora_mysql_8_replication_enabled(cluster):+ if compute_recreated:+ # A new container has a new hostname, while the+ # persisted replica repository and relay-log names+ # belong to the old container. Reset them before+ # changing the replication source.+ cluster["_mysql_replication_reset_pending"] = True+ _configure_or_defer_mysql_replication(cluster_id, cluster)+ if not cluster.get("_shared_container_ready"):+ _sync_cluster_endpoints(cluster)+ _refresh_cluster_status(cluster_id)+ returnfor inst in _cluster_member_instances(cluster):_attach_instance_to_shared_cluster(inst, cluster)inst["DBInstanceStatus"] = "available"@@ -5470,6 +5504,19 @@ def _stop_db_cluster(p):"https://github.com/ministackorg/ministack/issues/1325).",400,)+ global_cluster, _member = _global_cluster_member_for_cluster(cluster)+ if (+ global_cluster+ and len(global_cluster.get("GlobalClusterMembers", [])) > 1+ ):+ # Message text is verbatim from the AWS Aurora global-database+ # limitations documentation, not from a captured wire transcript.+ return _error(+ "InvalidDBClusterStateFault",+ "You can only stop and start a cluster that's part of an Aurora "+ "global database if it's the only cluster in the global database.",+ 400,+ )if not _stop_cluster_shared_container(cluster.get("DBClusterIdentifier", cluster_id), cluster,):diff --git a/tests/test_rds.py b/tests/test_rds.pyindex 495ca6c7..708acd6d 100644--- a/tests/test_rds.py+++ b/tests/test_rds.py@@ -2368,6 +2368,275 @@ def _poll_until(predicate, timeout=5):return predicate()+def test_rds_stop_db_cluster_refuses_nonsole_global_member(monkeypatch):+ """A member cannot stop while another cluster remains in its global."""+ from ministack.services import rds as m++ stopped = []+ monkeypatch.setattr(+ m,+ "_stop_cluster_shared_container",+ lambda *args: stopped.append(args) or True,+ )+ m._clusters.clear()+ m._global_clusters.clear()+ try:+ m._create_db_cluster({+ "DBClusterIdentifier": "global-stop-primary",+ "Engine": "aurora-mysql",+ })+ cluster = m._clusters.get("global-stop-primary")+ cluster["Status"] = "available"+ global_cluster = {+ "GlobalClusterIdentifier": "global-stop",+ "GlobalClusterMembers": [+ m._global_cluster_member(cluster, True),+ {"DBClusterArn": "arn:aws:rds:us-west-2:000000000000:cluster:secondary"},+ ],+ }+ m._global_clusters["global-stop"] = global_cluster+ cluster["GlobalClusterIdentifier"] = "global-stop"++ status, _, body = m._stop_db_cluster({+ "DBClusterIdentifier": "global-stop-primary",+ })++ assert status == 400+ assert b"InvalidDBClusterStateFault" in body+ assert (+ b"You can only stop and start a cluster that's part of an Aurora "+ b"global database if it's the only cluster in the global database."+ in body+ )+ assert stopped == []+ assert cluster["Status"] == "available"+ finally:+ m._global_clusters.clear()+ m._clusters.clear()+++def test_rds_stop_db_cluster_allows_sole_global_member(monkeypatch):+ """The sole cluster in a global database remains stoppable."""+ from ministack.services import rds as m++ monkeypatch.setattr(m, "_stop_cluster_shared_container", lambda *_args: True)+ m._clusters.clear()+ m._global_clusters.clear()+ try:+ m._create_db_cluster({+ "DBClusterIdentifier": "sole-global-member",+ "Engine": "aurora-mysql",+ })+ cluster = m._clusters.get("sole-global-member")+ cluster["Status"] = "available"+ global_cluster = {+ "GlobalClusterIdentifier": "sole-global",+ "GlobalClusterMembers": [m._global_cluster_member(cluster, True)],+ }+ m._global_clusters["sole-global"] = global_cluster+ cluster["GlobalClusterIdentifier"] = "sole-global"++ status, _, _body = m._stop_db_cluster({+ "DBClusterIdentifier": "sole-global-member",+ })++ assert status == 200+ assert cluster["Status"] == "stopped"+ finally:+ m._global_clusters.clear()+ m._clusters.clear()+++def test_rds_stop_db_cluster_postgres_fails_closed_on_missing_backref(monkeypatch):+ """Global membership guards are engine-agnostic and fail closed."""+ from ministack.services import rds as m++ stopped = []+ monkeypatch.setattr(+ m, "_stop_cluster_shared_container",+ lambda *args: stopped.append(args) or True,+ )+ m._clusters.clear()+ m._global_clusters.clear()+ try:+ m._create_db_cluster({+ "DBClusterIdentifier": "postgres-global-primary",+ "Engine": "aurora-postgresql",+ })+ cluster = m._clusters["postgres-global-primary"]+ cluster["Status"] = "available"+ cluster["GlobalClusterIdentifier"] = "postgres-global"+ m._global_clusters["postgres-global"] = {+ "GlobalClusterIdentifier": "postgres-global",+ # Deliberately omit this cluster's ARN to exercise a stale+ # GlobalClusterMembers back-reference.+ "GlobalClusterMembers": [+ {"DBClusterArn": "arn:aws:rds:us-west-2:0:cluster:secondary"},+ {"DBClusterArn": "arn:aws:rds:eu-west-1:0:cluster:tertiary"},+ ],+ }++ status, _, body = m._stop_db_cluster({+ "DBClusterIdentifier": "postgres-global-primary",+ })++ assert status == 400+ assert b"InvalidDBClusterStateFault" in body+ assert stopped == []+ finally:+ m._global_clusters.clear()+ m._clusters.clear()+++def test_rds_start_db_cluster_global_membership_guard(monkeypatch):+ """Start refuses multi-member globals but allows a sole member."""+ from ministack.services import rds as m++ monkeypatch.setattr(m, "_get_docker", lambda: None)+ m._clusters.clear()+ m._global_clusters.clear()+ try:+ m._create_db_cluster({+ "DBClusterIdentifier": "global-start-primary",+ "Engine": "aurora-mysql",+ })+ cluster = m._clusters["global-start-primary"]+ cluster["Status"] = "stopped"+ cluster["GlobalClusterIdentifier"] = "global-start"+ global_cluster = {+ "GlobalClusterIdentifier": "global-start",+ "GlobalClusterMembers": [+ m._global_cluster_member(cluster, True),+ {"DBClusterArn": "arn:aws:rds:us-west-2:0:cluster:secondary"},+ ],+ }+ m._global_clusters["global-start"] = global_cluster++ status, _, body = m._start_db_cluster({+ "DBClusterIdentifier": "global-start-primary",+ })+ assert status == 400+ assert b"InvalidDBClusterStateFault" in body+ assert cluster["Status"] == "stopped"++ global_cluster["GlobalClusterMembers"] = [+ m._global_cluster_member(cluster, True),+ ]+ status, _, body = m._start_db_cluster({+ "DBClusterIdentifier": "global-start-primary",+ })+ assert status == 200, body+ assert cluster["Status"] == "available"+ finally:+ m._global_clusters.clear()+ m._clusters.clear()+++ ("recreate", "legacy_volume", "expected_reset", "expected_status"),+ [+ (True, False, True, "available"),+ (False, False, None, "available"),+ (True, True, True, "failed"),+ ],+)+def test_rds_start_reconfigures_global_mysql_replication(+ monkeypatch, recreate, legacy_volume, expected_reset, expected_status,+):+ """Start relinks global MySQL with the right recreated-container state."""+ from ministack.services import rds as m++ docker, _container, state = _stop_start_fake_docker(+ m._rds_cluster_docker_name("recreated-global-cluster"),+ )+ replication_states = []+ monkeypatch.setattr(m, "_get_docker", lambda: docker)+ monkeypatch.setattr(m, "_get_ministack_network", lambda _client: None)+ monkeypatch.setattr(m, "_next_port", lambda: 16076)+ monkeypatch.setattr(m, "_is_host_port_free", lambda _port: True)+ monkeypatch.setattr(m, "_wait_for_database_ready", lambda *_args: True)+ monkeypatch.setattr(m, "_ensure_mysql_compatibility", lambda *_args, **_kwargs: True)+ monkeypatch.setattr(m, "_grant_mysql_master_user_privileges", lambda *_args: None)+ original_configure = m._configure_or_defer_mysql_replication++ def configure_inner(_cluster_id, cluster):+ if legacy_volume:+ cluster["_shared_container_ready"] = False+ m._set_cluster_members_status(cluster, "failed")+ return None++ def capture_configure(cluster_id, cluster):+ replication_states.append((+ cluster.get("_shared_container_epoch"),+ cluster.get("_shared_container_ready"),+ cluster.get("_mysql_replication_reset_pending"),+ ))+ return original_configure(cluster_id, cluster)++ monkeypatch.setattr(m, "_configure_mysql_replication", configure_inner)+ monkeypatch.setattr(+ m,+ "_configure_or_defer_mysql_replication",+ capture_configure,+ )++ m._instances.clear()+ m._clusters.clear()+ m._global_clusters.clear()+ try:+ m._create_db_cluster({+ "DBClusterIdentifier": "recreated-global-cluster",+ "Engine": "aurora-mysql",+ "MasterUsername": "admin",+ "MasterUserPassword": "password123",+ })+ cluster = m._clusters.get("recreated-global-cluster")+ cluster["Status"] = "stopped"+ cluster["_shared_container_ready"] = False+ cluster["_shared_storage_initialized"] = True+ cluster["_shared_container_id"] = docker.containers.get(+ m._rds_cluster_docker_name("recreated-global-cluster"),+ ).id+ cluster["DBClusterMembers"] = [{+ "DBInstanceIdentifier": "recreated-global-instance",+ "IsClusterWriter": True,+ }]+ m._instances["recreated-global-instance"] = {+ "DBInstanceIdentifier": "recreated-global-instance",+ "DBInstanceStatus": "stopped",+ }+ global_cluster = {+ "GlobalClusterIdentifier": "recreated-global",+ "GlobalClusterMembers": [m._global_cluster_member(cluster, True)],+ }+ m._global_clusters["recreated-global"] = global_cluster+ cluster["GlobalClusterIdentifier"] = "recreated-global"+ state["fail_start"] = recreate++ status, _, _body = m._start_db_cluster({+ "DBClusterIdentifier": "recreated-global-cluster",+ })++ assert status == 200+ assert _poll_until(+ lambda: m._instances["recreated-global-instance"][+ "DBInstanceStatus"+ ] == expected_status,+ )+ assert replication_states == [(+ cluster["_shared_container_epoch"],+ True,+ expected_reset,+ )]+ assert m._instances["recreated-global-instance"][+ "DBInstanceStatus"+ ] == expected_status+ finally:+ m._global_clusters.clear()+ m._clusters.clear()+ m._instances.clear()++def test_rds_start_db_cluster_accepts_cluster_arn(monkeypatch):"""StartDBCluster by ARN must reach available, not wedge in starting."""from ministack.services import rds as m@@ -8422,6 +8691,80 @@ def test_rds_deleting_last_global_secondary_instance_preserves_headless_applier(m._global_clusters.clear()+def test_rds_deleting_last_global_primary_instance_preserves_compute(monkeypatch):+ """A global writer keeps compute while secondaries still depend on it."""+ from ministack.core.responses import get_account_id+ from ministack.services import rds as m++ account_id = get_account_id()+ writer, secondary, _writer_member, _secondary_member, global_cluster = (+ _mysql_replication_unit_topology()+ )+ writer.update({+ "DBClusterMembers": [{+ "DBInstanceIdentifier": "primary-writer",+ "IsClusterWriter": True,+ }],+ "_shared_storage_initialized": True,+ "_shared_container_id": "primary-container",+ })+ instance = {+ "DBInstanceIdentifier": "primary-writer",+ "DBInstanceClass": "db.r6g.large",+ "Engine": "aurora-mysql",+ "EngineVersion": DEFAULT_AURORA_MYSQL_ENGINE_VERSION,+ "DBInstanceStatus": "available",+ "MasterUsername": "admin",+ "AllocatedStorage": 1,+ "DBInstanceArn": f"arn:aws:rds:us-east-1:{account_id}:db:primary-writer",+ "DBClusterIdentifier": "primary",+ "_shared_cluster_id": "primary",+ "DeletionProtection": False,+ }+ stopped = []+ secondary_state = {+ key: secondary.get(key)+ for key in (+ "_shared_container_ready",+ "_mysql_replication_source_arn",+ "_mysql_replication_reset_pending",+ )+ }++ m._instances.clear()+ m._clusters.clear()+ m._global_clusters.clear()+ try:+ m._clusters.set_scoped(account_id, "us-east-1", "primary", writer)+ m._clusters.set_scoped(account_id, "us-west-2", "secondary", secondary)+ m._instances.set_scoped(+ account_id, "us-east-1", "primary-writer", instance,+ )+ m._global_clusters["global-repl"] = global_cluster+ monkeypatch.setattr(+ m,+ "_stop_cluster_shared_container",+ lambda cluster_id, cluster: stopped.append((cluster_id, cluster)),+ )++ status, _, body = m._delete_db_instance({+ "DBInstanceIdentifier": "primary-writer",+ "SkipFinalSnapshot": "true",+ })++ assert status == 200, body+ assert stopped == []+ assert writer["DBClusterMembers"] == []+ assert writer["_shared_container_id"] == "primary-container"+ assert {+ key: secondary.get(key) for key in secondary_state+ } == secondary_state+ finally:+ m._instances.clear()+ m._clusters.clear()+ m._global_clusters.clear()++def test_rds_restore_respawns_persisted_headless_secondary_applier(monkeypatch):import threading