Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

rds: enforce global-database Stop restriction and re-link replication on Start (#1438)

kiran01bmc3f09b739481
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9b3be612..037c01ad 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,8 @@ Versioning follows [Semantic Versioning](https://semver.org/).
## [Unreleased]
+### Fixed
+- **RDS — global Aurora stop/start preserves topology and MySQL replication** — `StopDBCluster` and `StartDBCluster` allowed individual members of a multi-cluster Aurora global database to transition independently, and deleting a primary's last instance stopped shared compute that secondaries still needed. Starts and stops are now limited to sole-member global databases (`InvalidDBClusterStateFault`, 400), last-instance deletion preserves compute while other global members exist, and a successful recreate resets stale MySQL replica state before reconfiguring or deferring replication after readiness without overwriting a deliberate legacy-volume failure. Contributed by @kiran01bm.
### Added
- **IoT Core — native mTLS MQTT listener on port 8883** — the embedded broker spoke MQTT only over WebSocket, so AWS IoT Device SDK binaries could not connect. The broker now also listens for MQTT over TLS on port 8883 (`IOT_MTLS_ENABLED=0` turns it off, `IOT_MTLS_PORT` moves it), on by default when `cryptography` is available, configured exactly like the Transfer Family SFTP listener. The broker certificate is minted from the local CA (`GET /_ministack/iot/ca.pem`). The client certificate is optional and follows the rule S3 applies to a presigned URL: a client that presents none is served under `MINISTACK_ACCOUNT_ID`, exactly like an unsigned MQTT-over-WebSocket upgrade, while one that presents a certificate has it read against the registry. Registered `ACTIVE` in exactly one account serves that account and region; unknown, not `ACTIVE`, or `ACTIVE` in several accounts is refused with a "not authorized" CONNACK (0x05, reason code 0x87 for MQTT 5 clients), so `UpdateCertificate` to `INACTIVE` cuts a device off the way it does on AWS. Contributed by @iot-rocket.
diff --git a/ministack/services/rds.py b/ministack/services/rds.py
index 07ee791c..81cd6e4d 100644
--- a/ministack/services/rds.py
+++ b/ministack/services/rds.py
@@ -3679,12 +3679,17 @@ def _delete_db_instance(p):
if shared_cluster_id:
cluster = _resolve_cluster_in_request_region(shared_cluster_id)
if cluster and not cluster.get("DBClusterMembers"):
- if _mysql_replication_secondary(cluster):
+ global_cluster, _member = _global_cluster_member_for_cluster(cluster)
+ if (
+ global_cluster
+ and len(global_cluster.get("GlobalClusterMembers", [])) > 1
+ ):
# Aurora global headless secondaries have no query compute but
- # their storage continues to synchronize. MiniStack's single
- # shared process is also the applier, so preserve it until the
- # secondary is detached, deleted, or gains compute again.
- cluster["_mysql_headless_applier_required"] = True
+ # their storage continues to synchronize. MiniStack's shared
+ # process also sustains the global topology, so preserve it
+ # for both writers and secondaries while other members exist.
+ if _mysql_replication_secondary(cluster):
+ cluster["_mysql_headless_applier_required"] = True
else:
_stop_cluster_shared_container(shared_cluster_id, cluster)
@@ -5226,6 +5231,19 @@ def _start_db_cluster(p):
"stopped,inaccessible-encryption-credentials-recoverable.",
400,
)
+ global_cluster, _member = _global_cluster_member_for_cluster(cluster)
+ if (
+ global_cluster
+ and len(global_cluster.get("GlobalClusterMembers", [])) > 1
+ ):
+ # Message text is verbatim from the AWS Aurora global-database
+ # limitations documentation, not from a captured wire transcript.
+ return _error(
+ "InvalidDBClusterStateFault",
+ "You can only stop and start a cluster that's part of an Aurora "
+ "global database if it's the only cluster in the global database.",
+ 400,
+ )
member_instances = _cluster_member_instances(cluster)
docker_client = _get_docker()
@@ -5242,6 +5260,7 @@ def _start_db_cluster(p):
return _xml(200, "StartDBClusterResponse",
f"<StartDBClusterResult><DBCluster>{_cluster_xml(cluster)}</DBCluster></StartDBClusterResult>")
+ compute_recreated = False
if had_compute:
start_result = _restart_cluster_shared_container(cluster_id, cluster)
if start_result.get("failed"):
@@ -5251,6 +5270,7 @@ def _start_db_cluster(p):
start_result = _start_cluster_shared_container(
cluster_id, cluster, remove_stale=True,
)
+ compute_recreated = start_result.get("started", False)
else:
# Initialized storage without a container id happens after a host
# restart: persistence drops the container id but the stable-named
@@ -5258,6 +5278,7 @@ def _start_db_cluster(p):
start_result = _start_cluster_shared_container(
cluster_id, cluster, remove_stale=True,
)
+ compute_recreated = start_result.get("started", False)
if not start_result.get("started"):
if start_result.get("failed"):
@@ -5301,6 +5322,7 @@ def _start_db_cluster(p):
container_epoch=start_result.get("container_epoch"),
ready_host=start_result.get("readiness_host"),
ready_port=start_result.get("readiness_port"),
+ compute_recreated=compute_recreated,
):
cluster = _clusters.get(cluster_id)
if not cluster:
@@ -5416,6 +5438,18 @@ def _start_db_cluster(p):
cluster_id,
)
cluster["_shared_container_ready"] = True
+ if _aurora_mysql_8_replication_enabled(cluster):
+ if compute_recreated:
+ # A new container has a new hostname, while the
+ # persisted replica repository and relay-log names
+ # belong to the old container. Reset them before
+ # changing the replication source.
+ cluster["_mysql_replication_reset_pending"] = True
+ _configure_or_defer_mysql_replication(cluster_id, cluster)
+ if not cluster.get("_shared_container_ready"):
+ _sync_cluster_endpoints(cluster)
+ _refresh_cluster_status(cluster_id)
+ return
for inst in _cluster_member_instances(cluster):
_attach_instance_to_shared_cluster(inst, cluster)
inst["DBInstanceStatus"] = "available"
@@ -5470,6 +5504,19 @@ def _stop_db_cluster(p):
"https://github.com/ministackorg/ministack/issues/1325).",
400,
)
+ global_cluster, _member = _global_cluster_member_for_cluster(cluster)
+ if (
+ global_cluster
+ and len(global_cluster.get("GlobalClusterMembers", [])) > 1
+ ):
+ # Message text is verbatim from the AWS Aurora global-database
+ # limitations documentation, not from a captured wire transcript.
+ return _error(
+ "InvalidDBClusterStateFault",
+ "You can only stop and start a cluster that's part of an Aurora "
+ "global database if it's the only cluster in the global database.",
+ 400,
+ )
if not _stop_cluster_shared_container(
cluster.get("DBClusterIdentifier", cluster_id), cluster,
):
diff --git a/tests/test_rds.py b/tests/test_rds.py
index 495ca6c7..708acd6d 100644
--- a/tests/test_rds.py
+++ b/tests/test_rds.py
@@ -2368,6 +2368,275 @@ def _poll_until(predicate, timeout=5):
return predicate()
+def test_rds_stop_db_cluster_refuses_nonsole_global_member(monkeypatch):
+ """A member cannot stop while another cluster remains in its global."""
+ from ministack.services import rds as m
+
+ stopped = []
+ monkeypatch.setattr(
+ m,
+ "_stop_cluster_shared_container",
+ lambda *args: stopped.append(args) or True,
+ )
+ m._clusters.clear()
+ m._global_clusters.clear()
+ try:
+ m._create_db_cluster({
+ "DBClusterIdentifier": "global-stop-primary",
+ "Engine": "aurora-mysql",
+ })
+ cluster = m._clusters.get("global-stop-primary")
+ cluster["Status"] = "available"
+ global_cluster = {
+ "GlobalClusterIdentifier": "global-stop",
+ "GlobalClusterMembers": [
+ m._global_cluster_member(cluster, True),
+ {"DBClusterArn": "arn:aws:rds:us-west-2:000000000000:cluster:secondary"},
+ ],
+ }
+ m._global_clusters["global-stop"] = global_cluster
+ cluster["GlobalClusterIdentifier"] = "global-stop"
+
+ status, _, body = m._stop_db_cluster({
+ "DBClusterIdentifier": "global-stop-primary",
+ })
+
+ assert status == 400
+ assert b"InvalidDBClusterStateFault" in body
+ assert (
+ b"You can only stop and start a cluster that's part of an Aurora "
+ b"global database if it's the only cluster in the global database."
+ in body
+ )
+ assert stopped == []
+ assert cluster["Status"] == "available"
+ finally:
+ m._global_clusters.clear()
+ m._clusters.clear()
+
+
+def test_rds_stop_db_cluster_allows_sole_global_member(monkeypatch):
+ """The sole cluster in a global database remains stoppable."""
+ from ministack.services import rds as m
+
+ monkeypatch.setattr(m, "_stop_cluster_shared_container", lambda *_args: True)
+ m._clusters.clear()
+ m._global_clusters.clear()
+ try:
+ m._create_db_cluster({
+ "DBClusterIdentifier": "sole-global-member",
+ "Engine": "aurora-mysql",
+ })
+ cluster = m._clusters.get("sole-global-member")
+ cluster["Status"] = "available"
+ global_cluster = {
+ "GlobalClusterIdentifier": "sole-global",
+ "GlobalClusterMembers": [m._global_cluster_member(cluster, True)],
+ }
+ m._global_clusters["sole-global"] = global_cluster
+ cluster["GlobalClusterIdentifier"] = "sole-global"
+
+ status, _, _body = m._stop_db_cluster({
+ "DBClusterIdentifier": "sole-global-member",
+ })
+
+ assert status == 200
+ assert cluster["Status"] == "stopped"
+ finally:
+ m._global_clusters.clear()
+ m._clusters.clear()
+
+
+def test_rds_stop_db_cluster_postgres_fails_closed_on_missing_backref(monkeypatch):
+ """Global membership guards are engine-agnostic and fail closed."""
+ from ministack.services import rds as m
+
+ stopped = []
+ monkeypatch.setattr(
+ m, "_stop_cluster_shared_container",
+ lambda *args: stopped.append(args) or True,
+ )
+ m._clusters.clear()
+ m._global_clusters.clear()
+ try:
+ m._create_db_cluster({
+ "DBClusterIdentifier": "postgres-global-primary",
+ "Engine": "aurora-postgresql",
+ })
+ cluster = m._clusters["postgres-global-primary"]
+ cluster["Status"] = "available"
+ cluster["GlobalClusterIdentifier"] = "postgres-global"
+ m._global_clusters["postgres-global"] = {
+ "GlobalClusterIdentifier": "postgres-global",
+ # Deliberately omit this cluster's ARN to exercise a stale
+ # GlobalClusterMembers back-reference.
+ "GlobalClusterMembers": [
+ {"DBClusterArn": "arn:aws:rds:us-west-2:0:cluster:secondary"},
+ {"DBClusterArn": "arn:aws:rds:eu-west-1:0:cluster:tertiary"},
+ ],
+ }
+
+ status, _, body = m._stop_db_cluster({
+ "DBClusterIdentifier": "postgres-global-primary",
+ })
+
+ assert status == 400
+ assert b"InvalidDBClusterStateFault" in body
+ assert stopped == []
+ finally:
+ m._global_clusters.clear()
+ m._clusters.clear()
+
+
+def test_rds_start_db_cluster_global_membership_guard(monkeypatch):
+ """Start refuses multi-member globals but allows a sole member."""
+ from ministack.services import rds as m
+
+ monkeypatch.setattr(m, "_get_docker", lambda: None)
+ m._clusters.clear()
+ m._global_clusters.clear()
+ try:
+ m._create_db_cluster({
+ "DBClusterIdentifier": "global-start-primary",
+ "Engine": "aurora-mysql",
+ })
+ cluster = m._clusters["global-start-primary"]
+ cluster["Status"] = "stopped"
+ cluster["GlobalClusterIdentifier"] = "global-start"
+ global_cluster = {
+ "GlobalClusterIdentifier": "global-start",
+ "GlobalClusterMembers": [
+ m._global_cluster_member(cluster, True),
+ {"DBClusterArn": "arn:aws:rds:us-west-2:0:cluster:secondary"},
+ ],
+ }
+ m._global_clusters["global-start"] = global_cluster
+
+ status, _, body = m._start_db_cluster({
+ "DBClusterIdentifier": "global-start-primary",
+ })
+ assert status == 400
+ assert b"InvalidDBClusterStateFault" in body
+ assert cluster["Status"] == "stopped"
+
+ global_cluster["GlobalClusterMembers"] = [
+ m._global_cluster_member(cluster, True),
+ ]
+ status, _, body = m._start_db_cluster({
+ "DBClusterIdentifier": "global-start-primary",
+ })
+ assert status == 200, body
+ assert cluster["Status"] == "available"
+ finally:
+ m._global_clusters.clear()
+ m._clusters.clear()
+
+
+ ("recreate", "legacy_volume", "expected_reset", "expected_status"),
+ [
+ (True, False, True, "available"),
+ (False, False, None, "available"),
+ (True, True, True, "failed"),
+ ],
+)
+def test_rds_start_reconfigures_global_mysql_replication(
+ monkeypatch, recreate, legacy_volume, expected_reset, expected_status,
+):
+ """Start relinks global MySQL with the right recreated-container state."""
+ from ministack.services import rds as m
+
+ docker, _container, state = _stop_start_fake_docker(
+ m._rds_cluster_docker_name("recreated-global-cluster"),
+ )
+ replication_states = []
+ monkeypatch.setattr(m, "_get_docker", lambda: docker)
+ monkeypatch.setattr(m, "_get_ministack_network", lambda _client: None)
+ monkeypatch.setattr(m, "_next_port", lambda: 16076)
+ monkeypatch.setattr(m, "_is_host_port_free", lambda _port: True)
+ monkeypatch.setattr(m, "_wait_for_database_ready", lambda *_args: True)
+ monkeypatch.setattr(m, "_ensure_mysql_compatibility", lambda *_args, **_kwargs: True)
+ monkeypatch.setattr(m, "_grant_mysql_master_user_privileges", lambda *_args: None)
+ original_configure = m._configure_or_defer_mysql_replication
+
+ def configure_inner(_cluster_id, cluster):
+ if legacy_volume:
+ cluster["_shared_container_ready"] = False
+ m._set_cluster_members_status(cluster, "failed")
+ return None
+
+ def capture_configure(cluster_id, cluster):
+ replication_states.append((
+ cluster.get("_shared_container_epoch"),
+ cluster.get("_shared_container_ready"),
+ cluster.get("_mysql_replication_reset_pending"),
+ ))
+ return original_configure(cluster_id, cluster)
+
+ monkeypatch.setattr(m, "_configure_mysql_replication", configure_inner)
+ monkeypatch.setattr(
+ m,
+ "_configure_or_defer_mysql_replication",
+ capture_configure,
+ )
+
+ m._instances.clear()
+ m._clusters.clear()
+ m._global_clusters.clear()
+ try:
+ m._create_db_cluster({
+ "DBClusterIdentifier": "recreated-global-cluster",
+ "Engine": "aurora-mysql",
+ "MasterUsername": "admin",
+ "MasterUserPassword": "password123",
+ })
+ cluster = m._clusters.get("recreated-global-cluster")
+ cluster["Status"] = "stopped"
+ cluster["_shared_container_ready"] = False
+ cluster["_shared_storage_initialized"] = True
+ cluster["_shared_container_id"] = docker.containers.get(
+ m._rds_cluster_docker_name("recreated-global-cluster"),
+ ).id
+ cluster["DBClusterMembers"] = [{
+ "DBInstanceIdentifier": "recreated-global-instance",
+ "IsClusterWriter": True,
+ }]
+ m._instances["recreated-global-instance"] = {
+ "DBInstanceIdentifier": "recreated-global-instance",
+ "DBInstanceStatus": "stopped",
+ }
+ global_cluster = {
+ "GlobalClusterIdentifier": "recreated-global",
+ "GlobalClusterMembers": [m._global_cluster_member(cluster, True)],
+ }
+ m._global_clusters["recreated-global"] = global_cluster
+ cluster["GlobalClusterIdentifier"] = "recreated-global"
+ state["fail_start"] = recreate
+
+ status, _, _body = m._start_db_cluster({
+ "DBClusterIdentifier": "recreated-global-cluster",
+ })
+
+ assert status == 200
+ assert _poll_until(
+ lambda: m._instances["recreated-global-instance"][
+ "DBInstanceStatus"
+ ] == expected_status,
+ )
+ assert replication_states == [(
+ cluster["_shared_container_epoch"],
+ True,
+ expected_reset,
+ )]
+ assert m._instances["recreated-global-instance"][
+ "DBInstanceStatus"
+ ] == expected_status
+ finally:
+ m._global_clusters.clear()
+ m._clusters.clear()
+ m._instances.clear()
+
+
def test_rds_start_db_cluster_accepts_cluster_arn(monkeypatch):
"""StartDBCluster by ARN must reach available, not wedge in starting."""
from ministack.services import rds as m
@@ -8422,6 +8691,80 @@ def test_rds_deleting_last_global_secondary_instance_preserves_headless_applier(
m._global_clusters.clear()
+def test_rds_deleting_last_global_primary_instance_preserves_compute(monkeypatch):
+ """A global writer keeps compute while secondaries still depend on it."""
+ from ministack.core.responses import get_account_id
+ from ministack.services import rds as m
+
+ account_id = get_account_id()
+ writer, secondary, _writer_member, _secondary_member, global_cluster = (
+ _mysql_replication_unit_topology()
+ )
+ writer.update({
+ "DBClusterMembers": [{
+ "DBInstanceIdentifier": "primary-writer",
+ "IsClusterWriter": True,
+ }],
+ "_shared_storage_initialized": True,
+ "_shared_container_id": "primary-container",
+ })
+ instance = {
+ "DBInstanceIdentifier": "primary-writer",
+ "DBInstanceClass": "db.r6g.large",
+ "Engine": "aurora-mysql",
+ "EngineVersion": DEFAULT_AURORA_MYSQL_ENGINE_VERSION,
+ "DBInstanceStatus": "available",
+ "MasterUsername": "admin",
+ "AllocatedStorage": 1,
+ "DBInstanceArn": f"arn:aws:rds:us-east-1:{account_id}:db:primary-writer",
+ "DBClusterIdentifier": "primary",
+ "_shared_cluster_id": "primary",
+ "DeletionProtection": False,
+ }
+ stopped = []
+ secondary_state = {
+ key: secondary.get(key)
+ for key in (
+ "_shared_container_ready",
+ "_mysql_replication_source_arn",
+ "_mysql_replication_reset_pending",
+ )
+ }
+
+ m._instances.clear()
+ m._clusters.clear()
+ m._global_clusters.clear()
+ try:
+ m._clusters.set_scoped(account_id, "us-east-1", "primary", writer)
+ m._clusters.set_scoped(account_id, "us-west-2", "secondary", secondary)
+ m._instances.set_scoped(
+ account_id, "us-east-1", "primary-writer", instance,
+ )
+ m._global_clusters["global-repl"] = global_cluster
+ monkeypatch.setattr(
+ m,
+ "_stop_cluster_shared_container",
+ lambda cluster_id, cluster: stopped.append((cluster_id, cluster)),
+ )
+
+ status, _, body = m._delete_db_instance({
+ "DBInstanceIdentifier": "primary-writer",
+ "SkipFinalSnapshot": "true",
+ })
+
+ assert status == 200, body
+ assert stopped == []
+ assert writer["DBClusterMembers"] == []
+ assert writer["_shared_container_id"] == "primary-container"
+ assert {
+ key: secondary.get(key) for key in secondary_state
+ } == secondary_state
+ finally:
+ m._instances.clear()
+ m._clusters.clear()
+ m._global_clusters.clear()
+
+
def test_rds_restore_respawns_persisted_headless_secondary_applier(monkeypatch):
import threading