Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

feat(rds): add planned Aurora MySQL switchover (#1621)

Ian Oberst57a98b686d08
diff --git a/ministack/services/rds.py b/ministack/services/rds.py
index 51d33a33..6c0eff8e 100644
--- a/ministack/services/rds.py
+++ b/ministack/services/rds.py
@@ -54,7 +54,7 @@ import time
from urllib.parse import parse_qs
from xml.sax.saxutils import escape as _esc
-from ministack.core import container_reaper
+from ministack.core import container_reaper, persistence
from ministack.core.arn import ArnParseError, parse_arn
from ministack.core.concurrency import resource_lock, run_offloop, spawn_background
from ministack.core.persistence import load_state
@@ -130,6 +130,8 @@ _MYSQL_CONTROL_USER = "rdsadmin"
_MYSQL_CONTROL_PASSWORD = "ministack-rds-control"
_MYSQL_REPLICATION_RETRY_ATTEMPTS = 60
_MYSQL_REPLICATION_RETRY_INTERVAL = 1
+_MYSQL_GLOBAL_SWITCHOVER_TIMEOUT = 30
+_MYSQL_GLOBAL_SWITCHOVER_STATE = "_mysql_global_writer_switch_state"
_MYSQL_TRANSACTION_STATES = frozenset({"RUNNING", "LOCK WAIT", "ROLLING BACK", "COMMITTING"})
_MYSQL_MODIFIED_TRANSACTION_QUERY = (
"SELECT i.TRX_ID, i.TRX_STATE, i.TRX_STARTED, i.TRX_MYSQL_THREAD_ID, "
@@ -391,6 +393,13 @@ def restore_state(data):
_db_proxies.update(data.get("db_proxies", {}))
_db_proxy_endpoints.update(data.get("db_proxy_endpoints", {}))
_global_clusters.update(data.get("global_clusters", {}))
+ for global_cluster in _global_clusters._data.values():
+ marker = global_cluster.get(_MYSQL_GLOBAL_SWITCHOVER_STATE)
+ if isinstance(marker, dict):
+ # A live request owner cannot survive a process restart. Any
+ # persisted in-progress operation therefore becomes an explicit
+ # repair state rather than being resumed from an unknown phase.
+ marker["state"] = "repair_required"
# Persistence contains every account, while AccountScopedDict.values()
# intentionally exposes only the active request account. Reconcile the
# raw persisted records so a warm boot cannot leave secondary metadata
@@ -808,6 +817,14 @@ def restore_state(data):
cluster_id,
engine=cluster.get("Engine", "aurora-mysql"),
)
+ repair_required = (
+ _mysql_global_writer_switch_repair_required(cluster)
+ )
+ repair_fence_verified = bool(
+ authenticated_ready
+ and repair_required
+ and _mysql_writer_fence_active(cluster) is True
+ )
with _shared_container_lock:
current_cluster = _clusters.get(cluster_id)
if (
@@ -820,6 +837,26 @@ def restore_state(data):
or cluster.get("_shared_container_id") != container_id
):
return
+ if repair_required:
+ if authenticated_ready and not repair_fence_verified:
+ logger.error(
+ "RDS: stopping restored MySQL compute for %s "
+ "because its repair fence could not be verified",
+ cluster_id,
+ )
+ if not _contain_cluster_compute_preserving_volume(
+ cluster_id, cluster,
+ ):
+ logger.critical(
+ "RDS: unfenced repair compute for %s could "
+ "not be contained",
+ cluster_id,
+ )
+ # Keep a verified-fenced server reachable only as
+ # repair compute. Do not advertise it as ready or run
+ # normal replication reconciliation against an
+ # uncertain channel topology.
+ authenticated_ready = False
pending_rotation = cluster.get(
"_pending_master_password_rotation",
)
@@ -882,7 +919,12 @@ def restore_state(data):
# that transition.
break
else:
- cluster["_shared_container_ready"] = status == "available"
+ if _mysql_global_writer_switch_repair_required(cluster):
+ cluster["_shared_container_ready"] = False
+ _set_cluster_members_status(cluster, "failed")
+ _refresh_cluster_status(cluster_id)
+ else:
+ cluster["_shared_container_ready"] = status == "available"
# Superseded member volumes are recovery copies until the adopted
# writer volume has passed an authenticated database readiness
@@ -1100,6 +1142,17 @@ def _start_cluster_shared_container(cluster_id, cluster, remove_stale=False):
"--log-replica-updates",
f"--binlog-expire-logs-seconds={_MYSQL_BINLOG_RETENTION_SECONDS}",
]
+ if (
+ _is_mysql_engine(engine)
+ and _mysql_global_writer_switch_repair_required(cluster)
+ ):
+ # SET GLOBAL fences do not reliably survive a container restart. A
+ # persisted repair marker therefore has to fence the server before it
+ # accepts its first connection, not after readiness publishes it.
+ container_kwargs.setdefault("command", []).extend([
+ "--read-only=ON",
+ "--super-read-only=ON",
+ ])
if ms_network:
container_kwargs["network"] = ms_network
# Aurora storage belongs to the cluster, not to any member instance. Use a
@@ -1860,6 +1913,109 @@ def _stop_cluster_shared_container(cluster_id, cluster):
return True
+def _force_remove_cluster_compute_preserving_volume(cluster_id, cluster):
+ """Force-remove owned cluster compute without deleting durable storage."""
+ parsed = _parse_rds_arn(cluster.get("DBClusterArn", ""))
+ if not parsed or parsed[1] != "cluster":
+ logger.error(
+ "RDS: cannot prove repair compute scope for cluster %s",
+ cluster_id,
+ )
+ return False
+ scope, _resource_type, parsed_cluster_id = parsed
+ if parsed_cluster_id != cluster_id:
+ logger.error(
+ "RDS: repair compute identifier %s does not match ARN for %s",
+ cluster_id,
+ parsed_cluster_id,
+ )
+ return False
+ docker_client = _get_docker()
+ if not docker_client:
+ return False
+ identifiers = []
+ if cluster.get("_shared_container_id"):
+ identifiers.append(cluster["_shared_container_id"])
+ container_name = _rds_cluster_docker_name(
+ cluster_id, scope.account_id, scope.region,
+ )
+ if container_name not in identifiers:
+ identifiers.append(container_name)
+
+ def _not_found(error):
+ response = getattr(error, "response", None)
+ return (
+ getattr(error, "status_code", None) == 404
+ or getattr(response, "status_code", None) == 404
+ )
+
+ for identifier in identifiers:
+ try:
+ container = docker_client.containers.get(identifier)
+ except Exception as e:
+ if _not_found(e):
+ continue
+ logger.error(
+ "RDS: failed to inspect repair container %s for cluster %s: %s",
+ identifier,
+ cluster_id,
+ e,
+ )
+ return False
+ if not _rds_container_is_owned_by(
+ container,
+ expected_cluster_ids={cluster_id},
+ account_id=scope.account_id,
+ region=scope.region,
+ ):
+ logger.error(
+ "RDS: refusing to force-remove unowned repair container %s "
+ "for cluster %s",
+ identifier,
+ cluster_id,
+ )
+ return False
+ try:
+ container.remove(force=True, v=False)
+ except Exception as e:
+ logger.error(
+ "RDS: failed to force-remove repair container %s for cluster "
+ "%s: %s",
+ identifier,
+ cluster_id,
+ e,
+ )
+ return False
+ for identifier in identifiers:
+ try:
+ docker_client.containers.get(identifier)
+ except Exception as e:
+ if _not_found(e):
+ continue
+ logger.error(
+ "RDS: failed to verify repair container %s removal for cluster "
+ "%s: %s",
+ identifier,
+ cluster_id,
+ e,
+ )
+ return False
+ logger.error(
+ "RDS: repair container %s for cluster %s remains after force-remove",
+ identifier,
+ cluster_id,
+ )
+ return False
+ cluster["_shared_container_id"] = None
+ cluster["_shared_internal_address"] = None
+ cluster["_shared_internal_port"] = None
+ return True
+
+
+def _contain_cluster_compute_preserving_volume(cluster_id, cluster):
+ return _force_remove_cluster_compute_preserving_volume(cluster_id, cluster)
+
+
def _remove_cluster_shared_resources(
cluster_id, cluster, timeout=5, account_id=None, region=None,
):
@@ -2994,6 +3150,57 @@ def _ensure_mysql_replication_user(cluster, *, timeout=None, deadline=None):
return succeeded
+def _set_mysql_cluster_fenced(cluster, *, timeout=None, deadline=None):
+ """Install and verify a writer fence using bounded control connections."""
+ conn = None
+ succeeded = False
+ try:
+ if deadline is None and timeout is not None:
+ deadline = time.monotonic() + max(float(timeout), 0.0)
+ conn = _connect_mysql(
+ _mysql_replication_connection,
+ cluster,
+ _mysql_deadline_timeout(timeout=timeout, deadline=deadline),
+ )
+ if conn is None:
+ return False
+ succeeded = _set_mysql_writer_fence(conn, deadline=deadline)
+ if succeeded:
+ succeeded = _mysql_writer_fence_active(
+ cluster, deadline=deadline,
+ ) is True
+ except Exception as e:
+ logger.warning(
+ "RDS: failed to fence MySQL cluster %s: %s",
+ cluster.get("DBClusterIdentifier", "unknown"),
+ e,
+ )
+ finally:
+ if not _close_mysql_resources("cluster fence", connection=conn):
+ succeeded = False
+ return succeeded
+
+
+def _fence_or_contain_mysql_cluster(cluster, *, timeout=3):
+ """Return ``fenced`` or ``contained`` only after proving one is true."""
+ if _set_mysql_cluster_fenced(cluster, timeout=timeout):
+ return "fenced"
+ cluster_id = cluster.get("DBClusterIdentifier", "")
+ with _shared_container_lock:
+ contained = _contain_cluster_compute_preserving_volume(
+ cluster_id, cluster,
+ )
+ _set_cluster_members_status(cluster, "failed")
+ cluster["Status"] = "creating"
+ if contained:
+ return "contained"
+ logger.critical(
+ "RDS: unfenced MySQL compute for %s could not be contained",
+ cluster_id or "unknown",
+ )
+ return None
+
+
def _configure_mysql_replica_from_source(
cluster_id, replica, source, *, timeout=None, deadline=None,
):
@@ -3921,10 +4128,8 @@ def _resolve_instance(db_id):
def _set_cluster_members_status(cluster, status):
- for member in cluster.get("DBClusterMembers", []):
- instance = _instances.get(member.get("DBInstanceIdentifier"))
- if instance is not None:
- instance["DBInstanceStatus"] = status
+ for instance in _cluster_member_instances(cluster):
+ instance["DBInstanceStatus"] = status
def _attach_instance_to_shared_cluster(instance, cluster):
@@ -4979,7 +5184,7 @@ def _modify_db_instance(p):
return _single_instance_response("ModifyDBInstanceResponse", "ModifyDBInstanceResult", instance)
-def _start_db_instance(p):
+def _set_db_instance_status(p, status, response_name, result_name):
db_id = _p(p, "DBInstanceIdentifier")
instance = _resolve_instance(db_id)
if not instance:
@@ -4987,32 +5192,36 @@ def _start_db_instance(p):
if invalid_arn:
return invalid_arn
return _error("DBInstanceNotFound", f"DBInstance {db_id} not found.", 404)
- instance["DBInstanceStatus"] = "available"
- return _single_instance_response("StartDBInstanceResponse", "StartDBInstanceResult", instance)
+ cluster_id = (
+ instance.get("_shared_cluster_id") or instance.get("DBClusterIdentifier")
+ )
+ cluster = _resolve_cluster_in_request_region(cluster_id) if cluster_id else None
+ if cluster and (switch_error := _begin_mysql_global_instance_mutation(cluster)):
+ return switch_error
+ try:
+ instance["DBInstanceStatus"] = status
+ return _single_instance_response(response_name, result_name, instance)
+ finally:
+ if cluster:
+ _end_mysql_global_instance_mutation(cluster)
+
+
+def _start_db_instance(p):
+ return _set_db_instance_status(
+ p, "available", "StartDBInstanceResponse", "StartDBInstanceResult",
+ )
def _stop_db_instance(p):
- db_id = _p(p, "DBInstanceIdentifier")
- instance = _resolve_instance(db_id)
- if not instance:
- invalid_arn = _invalid_db_instance_identifier_error(db_id)
- if invalid_arn:
- return invalid_arn
- return _error("DBInstanceNotFound", f"DBInstance {db_id} not found.", 404)
- instance["DBInstanceStatus"] = "stopped"
- return _single_instance_response("StopDBInstanceResponse", "StopDBInstanceResult", instance)
+ return _set_db_instance_status(
+ p, "stopped", "StopDBInstanceResponse", "StopDBInstanceResult",
+ )
def _reboot_db_instance(p):
- db_id = _p(p, "DBInstanceIdentifier")
- instance = _resolve_instance(db_id)
- if not instance:
- invalid_arn = _invalid_db_instance_identifier_error(db_id)
- if invalid_arn:
- return invalid_arn
- return _error("DBInstanceNotFound", f"DBInstance {db_id} not found.", 404)
- instance["DBInstanceStatus"] = "available"
- return _single_instance_response("RebootDBInstanceResponse", "RebootDBInstanceResult", instance)
+ return _set_db_instance_status(
+ p, "available", "RebootDBInstanceResponse", "RebootDBInstanceResult",
+ )
# ---------------------------------------------------------------------------
@@ -7223,8 +7432,8 @@ def _invalid_global_cluster_identifier_error(global_id):
# Global Clusters
#
# Emulation scope: Aurora Global Database membership plus native MySQL 8 GTID
-# replication between provisioned members. Failover and switchover remain
-# metadata-only; their data-plane promotion behavior is a separate phase.
+# replication between provisioned members. Planned Aurora MySQL 8 switchovers
+# move the data plane synchronously; lossy failover remains metadata-only.
# ---------------------------------------------------------------------------
def _create_global_cluster(p):
@@ -7492,11 +7701,113 @@ def _find_global_cluster_target_member(gc, target_cluster_id):
return cluster, member
+def _resolve_global_writer_switch(p, *, exact_two=False):
+ gc_id = _p(p, "GlobalClusterIdentifier")
+ target_id = _p(p, "TargetDbClusterIdentifier")
+ invalid_id = _invalid_global_cluster_identifier_error(gc_id)
+ if invalid_id:
+ return None, invalid_id
+ gc = _resolve_global_cluster(gc_id)
+ if not gc:
+ return None, _error(
+ "GlobalClusterNotFoundFault", f"Global cluster {gc_id} not found.", 404,
+ )
+ if not target_id:
+ return None, _error("MissingParameter", "TargetDbClusterIdentifier is required", 400)
+ target, target_member = _find_global_cluster_target_member(gc, target_id)
+ if not target_member:
+ if not _resolve_cluster(target_id):
+ return None, _error(
+ "DBClusterNotFoundFault", f"DBCluster {target_id} not found.", 404,
+ )
+ return None, _error(
+ "InvalidGlobalClusterStateFault",
+ f"DBCluster {target_id} is not a secondary member of global "
+ f"cluster {gc_id}.",
+ 400,
+ )
+ members = gc.get("GlobalClusterMembers", [])
+ writer = next((member for member in members if member.get("IsWriter")), None)
+ wrong_size = len(members) != 2 if exact_two else len(members) < 2
+ if not writer or target_member.get("IsWriter") or wrong_size:
+ requirement = "one writer and one secondary" if exact_two else "a secondary"
+ return None, _error(
+ "InvalidGlobalClusterStateFault",
+ f"Global cluster {gc_id} must have {requirement} target to promote.",
+ 400,
+ )
+ return (gc, target, target_member, writer), None
+
+
+def _persisted_mysql_global_writer_switch_state(gc):
+ marker = gc.get(_MYSQL_GLOBAL_SWITCHOVER_STATE) if gc else None
+ if (
+ isinstance(marker, dict)
+ and marker.get("state") in ("in_progress", "repair_required")
+ ):
+ return marker
+ return None
+
+
+def _persisted_mysql_global_writer_switch_cluster(data, gc):
+ if not isinstance(data, dict):
+ return None
+ global_clusters = data.get("global_clusters")
+ if isinstance(global_clusters, (AccountScopedDict, AccountRegionScopedDict)):
+ candidates = global_clusters.values()
+ elif isinstance(global_clusters, dict):
+ candidates = global_clusters.values()
+ else:
+ return None
+ identifier = gc.get("GlobalClusterIdentifier")
+ arn = gc.get("GlobalClusterArn")
+ for candidate in candidates:
+ if not isinstance(candidate, dict):
+ continue
+ if candidate.get("GlobalClusterIdentifier") != identifier:
+ continue
+ if arn and candidate.get("GlobalClusterArn") != arn:
+ continue
+ return candidate
+ return None
+
+
+def _persisted_mysql_global_writer_switch_marker(data, gc):
+ persisted_gc = _persisted_mysql_global_writer_switch_cluster(data, gc)
+ if persisted_gc is None:
+ return None
+ return persisted_gc.get(_MYSQL_GLOBAL_SWITCHOVER_STATE)
+
+
+def _durably_persist_mysql_global_writer_switch_state(gc, expected_marker):
+ """Write and verify the recovery state around data-plane changes."""
+ if not persistence.PERSIST_STATE:
+ return True
+ persistence.save_state("rds", get_state())
+ persisted = persistence.load_state("rds")
+ persisted_gc = _persisted_mysql_global_writer_switch_cluster(persisted, gc)
+ if (
+ persisted_gc is not None
+ and persisted_gc.get(_MYSQL_GLOBAL_SWITCHOVER_STATE) == expected_marker
+ ):
+ return True
+ logger.error(
+ "RDS: global writer switchover recovery state could not be durably "
+ "persisted"
+ )
+ return False
+
+
+def _mysql_global_writer_switch_repair_required(cluster):
+ gc, _member = _global_cluster_member_for_cluster(cluster)
+ return _persisted_mysql_global_writer_switch_state(gc) is not None
+
+
def _active_mysql_global_writer_switch(gc):
owner = _mysql_global_writer_switch_owners.get(id(gc)) if gc else None
if owner and owner.get("global_cluster") is gc:
return owner
- return None
+ return _persisted_mysql_global_writer_switch_state(gc)
def _mysql_global_writer_switch_mutation_error(cluster):
@@ -7589,6 +7900,18 @@ def _mysql_global_writer_switch_ready(gc, source, target):
)
+def _mysql_global_writer_switch_owner(gc, source, target, operation_id):
+ return {
+ "global_cluster": gc,
+ "source": source,
+ "target": target,
+ "topology": _mysql_global_writer_switch_topology(gc),
+ "source_identity": _mysql_global_writer_switch_identity(source),
+ "target_identity": _mysql_global_writer_switch_identity(target),
+ "operation_id": operation_id,
+ }
+
+
def _claim_mysql_global_writer_switch(gc, source, target):
with _mysql_global_writer_switch_lock:
if _active_mysql_global_writer_switch(gc) or any(
@@ -7596,18 +7919,116 @@ def _claim_mysql_global_writer_switch(gc, source, target):
for cluster in (source, target)
):
return None
- owner = {
- "global_cluster": gc,
- "source": source,
- "target": target,
- "topology": _mysql_global_writer_switch_topology(gc),
- "source_identity": _mysql_global_writer_switch_identity(source),
- "target_identity": _mysql_global_writer_switch_identity(target),
+ operation_id = new_uuid()
+ owner = _mysql_global_writer_switch_owner(
+ gc, source, target, operation_id,
+ )
+ marker = {
+ "state": "in_progress",
+ "source_arn": source.get("DBClusterArn"),
+ "target_arn": target.get("DBClusterArn"),
+ "operation_id": operation_id,
}
+ gc[_MYSQL_GLOBAL_SWITCHOVER_STATE] = marker
+ if not _durably_persist_mysql_global_writer_switch_state(gc, marker):
+ gc.pop(_MYSQL_GLOBAL_SWITCHOVER_STATE, None)
+ return None
_mysql_global_writer_switch_owners[id(gc)] = owner
return owner
+def _mysql_global_writer_switch_repair_matches(marker, source, target):
+ return bool(
+ isinstance(marker, dict)
+ and marker.get("state") == "repair_required"
+ and marker.get("operation_id")
+ and marker.get("source_arn") == source.get("DBClusterArn")
+ and marker.get("target_arn") == target.get("DBClusterArn")
+ )
+
+
+def _mysql_global_writer_switch_repair_ready(gc, source, target, marker):
+ return bool(
+ _mysql_global_writer_switch_repair_matches(marker, source, target)
+ and gc.get(_MYSQL_GLOBAL_SWITCHOVER_STATE) is marker
+ and source.get("EngineVersion") == target.get("EngineVersion")
+ and _mysql_gtid_history_ready(source)
+ and _mysql_gtid_history_ready(target)
+ and all(
+ cluster.get("_shared_storage_initialized")
+ and cluster.get("DBClusterMembers")
+ and cluster.get("_shared_container_id")
+ and cluster.get("_shared_internal_address")
+ and cluster.get("_shared_internal_port")
+ and isinstance(cluster.get("_shared_container_epoch"), int)
+ and cluster.get("_shared_container_epoch") > 0
+ and not cluster.get("_pending_master_password_rotation")
+ and not _active_mysql_global_instance_mutation(cluster)
+ for cluster in (source, target)
+ )
+ )
+
+
+def _resume_mysql_global_writer_switch(gc, source, target, marker):
+ """Reclaim a restored repair marker for a retry of the same switchover."""
+ with _mysql_global_writer_switch_lock:
+ if id(gc) in _mysql_global_writer_switch_owners:
+ return None
+ if not _mysql_global_writer_switch_repair_ready(
+ gc, source, target, marker,
+ ):
+ return None
+ for cluster in (source, target):
+ cluster["_shared_container_ready"] = True
+ cluster["Status"] = "available"
+ _set_cluster_members_status(cluster, "available")
+ gc["Status"] = "available"
+ marker["state"] = "in_progress"
+ owner = _mysql_global_writer_switch_owner(
+ gc, source, target, marker["operation_id"],
+ )
+ _mysql_global_writer_switch_owners[id(gc)] = owner
+ return owner
+
+
+def _resolve_committed_mysql_global_writer_switch_repair(p):
+ gc = _resolve_global_cluster(_p(p, "GlobalClusterIdentifier"))
+ marker = _persisted_mysql_global_writer_switch_state(gc)
+ if not gc or not marker or marker.get("state") != "repair_required":
+ return None
+ members = gc.get("GlobalClusterMembers", [])
+ source_member = next(
+ (
+ member for member in members
+ if member.get("DBClusterArn") == marker.get("source_arn")
+ ),
+ None,
+ )
+ target_member = next(
+ (
+ member for member in members
+ if member.get("DBClusterArn") == marker.get("target_arn")
+ ),
+ None,
+ )
+ source = _resolve_global_member_cluster(source_member)
+ target = _resolve_global_member_cluster(target_member)
+ requested_target = _p(p, "TargetDbClusterIdentifier")
+ if (
+ not source
+ or not target
+ or not source_member
+ or not target_member
+ or source_member.get("IsWriter")
+ or not target_member.get("IsWriter")
+ or requested_target not in (
+ target.get("DBClusterIdentifier"), target.get("DBClusterArn"),
+ )
+ ):
+ return None
+ return gc, source, target, source_member, target_member, marker
+
+
def _mysql_global_writer_switch_still_owned(owner, expected_topology):
gc = owner["global_cluster"]
source = owner["source"]
@@ -7632,44 +8053,39 @@ def _finish_mysql_global_writer_switch(owner, *, repair_required=False):
return
if repair_required:
owner["state"] = "repair_required"
+ marker = gc.get(_MYSQL_GLOBAL_SWITCHOVER_STATE)
+ if not isinstance(marker, dict):
+ marker = {}
+ gc[_MYSQL_GLOBAL_SWITCHOVER_STATE] = marker
+ marker.update({
+ "state": "repair_required",
+ "source_arn": owner["source"].get("DBClusterArn"),
+ "target_arn": owner["target"].get("DBClusterArn"),
+ "operation_id": owner["operation_id"],
+ })
+ # The durable in-progress claim already restores as repair-required,
+ # so a failed update remains fail-closed across a restart.
+ _durably_persist_mysql_global_writer_switch_state(gc, marker)
+ return True
else:
- _mysql_global_writer_switch_owners.pop(id(gc), None)
+ gc.pop(_MYSQL_GLOBAL_SWITCHOVER_STATE, None)
gc["Status"] = "available"
+ if not _durably_persist_mysql_global_writer_switch_state(gc, None):
+ owner["state"] = "repair_required"
+ gc[_MYSQL_GLOBAL_SWITCHOVER_STATE] = {
+ "state": "repair_required",
+ "source_arn": owner["source"].get("DBClusterArn"),
+ "target_arn": owner["target"].get("DBClusterArn"),
+ "operation_id": owner["operation_id"],
+ }
+ return False
+ _mysql_global_writer_switch_owners.pop(id(gc), None)
+ return True
-def _switch_global_cluster_writer(p, *, allow_data_loss=False):
- gc_id = _p(p, "GlobalClusterIdentifier")
- target_cluster_id = _p(p, "TargetDbClusterIdentifier")
- invalid_id = _invalid_global_cluster_identifier_error(gc_id)
- if invalid_id:
- return invalid_id
- gc = _resolve_global_cluster(gc_id)
- if not gc:
- return _error("GlobalClusterNotFoundFault",
- f"Global cluster {gc_id} not found.", 404)
- if not target_cluster_id:
- return _error("MissingParameter", "TargetDbClusterIdentifier is required", 400)
-
- _target_cluster, target_member = _find_global_cluster_target_member(gc, target_cluster_id)
- if not target_member:
- cluster = _resolve_cluster(target_cluster_id)
- if not cluster:
- return _error("DBClusterNotFoundFault",
- f"DBCluster {target_cluster_id} not found.", 404)
- return _error("InvalidGlobalClusterStateFault",
- f"DBCluster {target_cluster_id} is not a secondary member of global cluster {gc_id}.", 400)
-
- members = gc.get("GlobalClusterMembers", [])
- current_writer = next((m for m in members if m.get("IsWriter")), None)
- if not current_writer or target_member.get("IsWriter") or len(members) < 2:
- return _error("InvalidGlobalClusterStateFault",
- f"Global cluster {gc_id} does not have a secondary target to promote.", 400)
-
- _set_global_cluster_writer(gc, target_member)
- gc["Status"] = "available"
-
+def _global_cluster_writer_switch_response(gc, current_writer, target_member, *, allow_data_loss):
response_gc = copy.deepcopy(gc)
- response_gc["Status"] = "switching-over" if not allow_data_loss else "failing-over"
+ response_gc["Status"] = "failing-over" if allow_data_loss else "switching-over"
response_gc["FailoverState"] = {
"Status": "pending",
"FromDbClusterArn": current_writer["DBClusterArn"],
@@ -7679,8 +8095,571 @@ def _switch_global_cluster_writer(p, *, allow_data_loss=False):
return response_gc
+def _wait_for_mysql_replication_healthy(replica, source, *, deadline):
+ """Wait for both reverse replication threads to reach the expected source."""
+ conn = None
+ cur = None
+ healthy = False
+ try:
+ conn = _connect_mysql(
+ _mysql_replication_connection,
+ replica,
+ _mysql_deadline_timeout(deadline=deadline),
+ )
+ if conn is None:
+ return False
+ cur = conn.cursor()
+ while True:
+ _execute_mysql(cur, "SHOW REPLICA STATUS", deadline=deadline)
+ row = cur.fetchone()
+ if isinstance(row, dict):
+ status = row
+ else:
+ description = getattr(cur, "description", None) or ()
+ columns = [column[0] for column in description]
+ status = dict(zip(columns, row or ()))
+ try:
+ source_port_matches = int(status.get("Source_Port")) == int(
+ source.get("_shared_internal_port"),
+ )
+ except (TypeError, ValueError):
+ source_port_matches = False
+ if (
+ status.get("Replica_IO_Running") == "Yes"
+ and status.get("Replica_SQL_Running") == "Yes"
+ and status.get("Source_Host") == source.get("_shared_internal_address")
+ and source_port_matches
+ ):
+ healthy = True
+ break
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ break
+ time.sleep(min(_MYSQL_REPLICATION_RETRY_INTERVAL, remaining))
+ except Exception as e:
+ logger.warning(
+ "RDS: reverse MySQL replication did not become healthy for %s: %s",
+ replica.get("DBClusterIdentifier", "unknown"), e,
+ )
+ finally:
+ if not _close_mysql_resources(
+ "replication health check", cursor=cur, connection=conn,
+ ):
+ healthy = False
+ return healthy
+
+
+def _rollback_mysql_global_writer_switch(
+ owner, *, target_channel_changed=False, reverse_channel_attempted=False,
+):
+ gc = owner["global_cluster"]
+ source = owner["source"]
+ target = owner["target"]
+ # Compensation has its own bounded budget so safety restoration is tried.
+ deadline = time.monotonic() + _MYSQL_GLOBAL_SWITCHOVER_TIMEOUT
+
+ safe = True
+ if reverse_channel_attempted:
+ try:
+ source_reset = bool(_reset_mysql_replication_channel(
+ source.get("DBClusterIdentifier", ""),
+ source,
+ clear_super_read_only=False,
+ allow_missing_channel=True,
+ deadline=deadline,
+ ))
+ if source_reset:
+ _clear_mysql_replication_metadata(source)
+ except Exception as e:
+ logger.warning("RDS: reverse channel rollback failed: %s", e)
+ source_reset = False
+ safe = safe and source_reset
+ if target_channel_changed:
+ try:
+ target_restored = bool(_configure_mysql_replica_from_source(
+ target.get("DBClusterIdentifier", ""), target, source,
+ deadline=deadline,
+ ))
+ if target_restored:
+ target_restored = _wait_for_mysql_replication_healthy(
+ target, source, deadline=deadline,
+ )
+ except Exception as e:
+ logger.warning("RDS: target channel rollback failed: %s", e)
+ target_restored = False
+ safe = safe and target_restored
+ try:
+ target_safety = _fence_or_contain_mysql_cluster(target)
+ safe = safe and target_safety == "fenced"
+ if safe:
+ if not _acquire_mysql_lifecycle_lock(deadline):
+ safe = False
+ else:
+ try:
+ safe = _mysql_global_writer_switch_still_owned(
+ owner, owner["topology"],
+ )
+ finally:
+ _shared_container_lock.release()
+ if safe:
+ safe = _set_mysql_cluster_writable(source, deadline=deadline)
+ except Exception as e:
+ logger.warning("RDS: global writer rollback failed: %s", e)
+ safe = False
+ finally:
+ # Repair ownership must be finalized even after compensation expires.
+ with _shared_container_lock:
+ owned = _active_mysql_global_writer_switch(gc) is owner
+ finalized = False
+ if owned:
+ finalized = _finish_mysql_global_writer_switch(
+ owner, repair_required=not safe,
+ )
+ if safe and owned and not finalized:
+ _fence_or_contain_mysql_cluster(source)
+ _fence_or_contain_mysql_cluster(target)
+ with _shared_container_lock:
+ if _active_mysql_global_writer_switch(gc) is owner:
+ _finish_mysql_global_writer_switch(
+ owner, repair_required=True,
+ )
+ return safe and owned and finalized
+
+
+def _planned_mysql_global_writer_switch_not_ready(target_cluster_id, message=None):
+ return _error(
+ "InvalidDBClusterStateFault",
+ message or f"DBCluster {target_cluster_id} is not ready for a "
+ "planned global writer switchover.",
+ 400,
+ )
+
+
+def _repair_committed_mysql_global_writer_switch(
+ p,
+ gc,
+ source,
+ target,
+ source_member,
+ target_member,
+ marker,
+ original_topology,
+ original_identities,
+ *,
+ deadline,
+):
+ """Finalize a retry whose durable metadata already names the new writer."""
+ gc_id = _p(p, "GlobalClusterIdentifier")
+ target_id = _p(p, "TargetDbClusterIdentifier")
+
+ if not _ensure_mysql_control_user(
+ source, require_quiescence_grants=True, deadline=deadline,
+ ) or not _ensure_mysql_control_user(target, deadline=deadline):
+ return _planned_mysql_global_writer_switch_not_ready(target_id)
+ if not _ensure_mysql_replication_user(target, deadline=deadline):
+ return _planned_mysql_global_writer_switch_not_ready(target_id)
+ if any(
+ _mysql_writer_fence_active(cluster, deadline=deadline) is not True
+ for cluster in (source, target)
+ ):
+ return _planned_mysql_global_writer_switch_not_ready(target_id)
+
+ if not _acquire_mysql_lifecycle_lock(deadline):
+ return _planned_mysql_global_writer_switch_not_ready(target_id)
+ try:
+ if (
+ _resolve_global_cluster(gc_id) is not gc
+ or _mysql_global_writer_switch_topology(gc) != original_topology
+ or not _mysql_global_writer_switch_repair_ready(
+ gc, source, target, marker,
+ )
+ or original_identities != tuple(
+ _mysql_global_writer_switch_identity(cluster)
+ for cluster in (source, target)
+ )
+ or _global_cluster_writer_cluster(gc) is not target
+ ):
+ return _planned_mysql_global_writer_switch_not_ready(
+ target_id,
+ f"Global cluster {gc_id} changed while preparing its "
+ "committed writer repair.",
+ )
+ owner = _resume_mysql_global_writer_switch(gc, source, target, marker)
+ if owner is None:
+ return _planned_mysql_global_writer_switch_not_ready(target_id)
+ finally:
+ _shared_container_lock.release()
+
+ try:
+ if not _configure_mysql_replica_from_source(
+ source.get("DBClusterIdentifier", ""), source, target,
+ deadline=deadline,
+ ):
+ raise RuntimeError("committed writer replica repair failed")
+ if not _wait_for_mysql_replication_healthy(
+ source, target, deadline=deadline,
+ ):
+ raise RuntimeError("committed writer replica did not become healthy")
+ if not _set_mysql_cluster_writable(target, deadline=deadline):
+ raise RuntimeError("committed writer could not be made writable")
+
+ source_fenced = _mysql_writer_fence_active(
+ source, deadline=deadline,
+ ) is True
+ target_writable = _mysql_control_query(
+ target, "SELECT @@GLOBAL.read_only, @@GLOBAL.super_read_only",
+ deadline=deadline,
+ ) == ((0, 0),)
+ if not _acquire_mysql_lifecycle_lock(deadline):
+ raise RuntimeError("committed writer repair lock deadline elapsed")
+ try:
+ if (
+ not source_fenced
+ or not target_writable
+ or not _mysql_global_writer_switch_still_owned(
+ owner, original_topology,
+ )
+ or _global_cluster_writer_cluster(gc) is not target
+ ):
+ raise RuntimeError("committed writer repair changed before finalization")
+ if not _finish_mysql_global_writer_switch(owner):
+ raise RuntimeError("committed writer repair could not be persisted")
+ result = _global_cluster_writer_switch_response(
+ gc,
+ source_member,
+ target_member,
+ allow_data_loss=False,
+ )
+ finally:
+ _shared_container_lock.release()
+ return result
+ except Exception as e:
+ logger.warning(
+ "RDS: committed global writer repair for %s failed: %s", gc_id, e,
+ )
+ _fence_or_contain_mysql_cluster(source)
+ _fence_or_contain_mysql_cluster(target)
+ with _shared_container_lock:
+ if _active_mysql_global_writer_switch(gc) is owner:
+ _finish_mysql_global_writer_switch(owner, repair_required=True)
+ return _planned_mysql_global_writer_switch_not_ready(target_id)
+
+
+def _planned_mysql_global_writer_switch(p):
+ """Synchronously exchange a two-member Aurora MySQL global writer."""
+ gc_id = _p(p, "GlobalClusterIdentifier")
+ target_cluster_id = _p(p, "TargetDbClusterIdentifier")
+
+ def _not_ready_error(message=None):
+ return _planned_mysql_global_writer_switch_not_ready(
+ target_cluster_id, message,
+ )
+
+ deadline = time.monotonic() + _MYSQL_GLOBAL_SWITCHOVER_TIMEOUT
+ if not _acquire_mysql_lifecycle_lock(deadline):
+ return _not_ready_error()
+ try:
+ committed_repair = _resolve_committed_mysql_global_writer_switch_repair(p)
+ if committed_repair:
+ (
+ gc,
+ source,
+ target,
+ current_writer,
+ target_member,
+ repair_marker,
+ ) = committed_repair
+ recovering = True
+ else:
+ resolved, error = _resolve_global_writer_switch(p)
+ if error:
+ return error
+ gc, target, target_member, current_writer = resolved
+ source = _resolve_global_member_cluster(current_writer)
+ if not source or not target:
+ return _not_ready_error()
+ repair_marker = _persisted_mysql_global_writer_switch_state(gc)
+ recovering = repair_marker is not None
+ live_owner = _mysql_global_writer_switch_owners.get(id(gc))
+ if live_owner and live_owner.get("global_cluster") is gc:
+ return _error(
+ "InvalidGlobalClusterStateFault",
+ f"Global cluster {gc_id} already has a writer switchover in "
+ "progress.",
+ 400,
+ )
+ if recovering and not _mysql_global_writer_switch_repair_matches(
+ repair_marker, source, target,
+ ):
+ return _error(
+ "InvalidGlobalClusterStateFault",
+ f"Global cluster {gc_id} requires repair by retrying its "
+ "interrupted writer switchover target.",
+ 400,
+ )
+ native = tuple(
+ _aurora_mysql_8_replication_enabled(cluster)
+ for cluster in (source, target)
+ )
+ if native == (False, False):
+ return _switch_global_cluster_writer_locked(
+ p, allow_data_loss=False,
+ )
+ if native != (True, True):
+ return _not_ready_error()
+ if len(gc.get("GlobalClusterMembers", [])) != 2:
+ return _not_ready_error()
+ if recovering:
+ if not _mysql_global_writer_switch_repair_ready(
+ gc, source, target, repair_marker,
+ ):
+ return _not_ready_error(
+ f"Global cluster {gc_id} repair compute is not ready to "
+ "retry its interrupted writer switchover.",
+ )
+ elif not _mysql_global_writer_switch_ready(gc, source, target):
+ return _not_ready_error()
+ original_topology = _mysql_global_writer_switch_topology(gc)
+ original_identities = tuple(
+ _mysql_global_writer_switch_identity(cluster)
+ for cluster in (source, target)
+ )
+ finally:
+ _shared_container_lock.release()
+
+ if committed_repair:
+ return _repair_committed_mysql_global_writer_switch(
+ p,
+ gc,
+ source,
+ target,
+ current_writer,
+ target_member,
+ repair_marker,
+ original_topology,
+ original_identities,
+ deadline=deadline,
+ )
+
+ def _remaining():
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise RuntimeError("global writer switchover deadline elapsed")
+ return remaining
+
+ if not _ensure_mysql_control_user(
+ source, require_quiescence_grants=True, deadline=deadline,
+ ) or not _ensure_mysql_control_user(target, deadline=deadline):
+ return _not_ready_error()
+ if not _ensure_mysql_replication_user(source, deadline=deadline):
+ return _not_ready_error()
+ if (
+ recovering
+ and _mysql_writer_fence_active(source, deadline=deadline) is not True
+ ):
+ return _not_ready_error()
+ if _mysql_writer_fence_active(target, deadline=deadline) is not True:
+ return _not_ready_error()
+
+ if not _acquire_mysql_lifecycle_lock(deadline):
+ return _not_ready_error()
+ try:
+ if (
+ _resolve_global_cluster(gc_id) is not gc
+ or _mysql_global_writer_switch_topology(gc) != original_topology
+ or _resolve_global_member_cluster(current_writer) is not source
+ or _resolve_global_member_cluster(target_member) is not target
+ or not (
+ _mysql_global_writer_switch_repair_ready(
+ gc, source, target, repair_marker,
+ )
+ if recovering else
+ _mysql_global_writer_switch_ready(gc, source, target)
+ )
+ or original_identities != tuple(
+ _mysql_global_writer_switch_identity(cluster)
+ for cluster in (source, target)
+ )
+ ):
+ return _not_ready_error(
+ f"Global cluster {gc_id} changed while preparing switchover.",
+ )
+ owner = (
+ _resume_mysql_global_writer_switch(
+ gc, source, target, repair_marker,
+ )
+ if recovering else
+ _claim_mysql_global_writer_switch(gc, source, target)
+ )
+ if owner is None:
+ if recovering:
+ return _not_ready_error(
+ f"Global cluster {gc_id} repair claim changed while "
+ "preparing its writer switchover retry.",
+ )
+ if not _active_mysql_global_writer_switch(gc):
+ return _not_ready_error(
+ f"Global cluster {gc_id} could not durably record its "
+ "planned writer switchover.",
+ )
+ return _error(
+ "InvalidGlobalClusterStateFault",
+ f"Global cluster {gc_id} already has a writer switchover in "
+ "progress.",
+ 400,
+ )
+ finally:
+ _shared_container_lock.release()
+
+ fence_conn = None
+ target_channel_changed = False
+ reverse_channel_attempted = False
+ metadata_flipped = False
+ try:
+ fence_conn = _mysql_replication_connection(source, timeout=_remaining())
+ if fence_conn is None or not _set_mysql_writer_fence(
+ fence_conn, deadline=deadline,
+ ):
+ raise RuntimeError("writer fence failed")
+ gtid = _wait_for_mysql_writer_quiescence(
+ source, _remaining(), deadline=deadline,
+ )
+ if gtid is None:
+ raise RuntimeError("writer drain failed")
+ if not _close_mysql_resources("switchover fence", connection=fence_conn):
+ raise RuntimeError("writer fence connection close failed")
+ fence_conn = None
+
+ if not _wait_for_mysql_gtid(
+ target, gtid, _remaining(), deadline=deadline,
+ ):
+ raise RuntimeError("target GTID convergence failed")
+
+ target_channel_changed = True
+ if not _reset_mysql_replication_channel(
+ target.get("DBClusterIdentifier", ""),
+ target,
+ clear_super_read_only=False,
+ allow_missing_channel=recovering,
+ deadline=deadline,
+ ):
+ raise RuntimeError("target replication reset failed")
+
+ reverse_channel_attempted = True
+ if not _configure_mysql_replica_from_source(
+ source.get("DBClusterIdentifier", ""),
+ source,
+ target,
+ deadline=deadline,
+ ):
+ raise RuntimeError("reverse replication configuration failed")
+ if not _wait_for_mysql_replication_healthy(
+ source, target, deadline=deadline,
+ ):
+ raise RuntimeError("reverse replication did not become healthy")
+
+ if not _acquire_mysql_lifecycle_lock(deadline):
+ raise RuntimeError("global writer commit lock deadline elapsed")
+ try:
+ if not _mysql_global_writer_switch_still_owned(
+ owner, owner["topology"],
+ ):
+ raise RuntimeError("global topology changed before commit")
+ _set_global_cluster_writer(gc, target_member)
+ metadata_flipped = True
+ committed_topology = _mysql_global_writer_switch_topology(gc)
+ _clear_mysql_replication_metadata(target)
+ _sync_global_mysql_credentials(target, gc)
+ finally:
+ _shared_container_lock.release()
+
+ if not _set_mysql_cluster_writable(target, deadline=deadline):
+ raise RuntimeError("promoted writer could not be made writable")
+
+ source_fenced = _mysql_writer_fence_active(source, deadline=deadline) is True
+ target_writable = _mysql_control_query(
+ target, "SELECT @@GLOBAL.read_only, @@GLOBAL.super_read_only",
+ deadline=deadline,
+ ) == ((0, 0),)
+
+ if not _acquire_mysql_lifecycle_lock(deadline):
+ raise RuntimeError("global writer finalization lock deadline elapsed")
+ try:
+ if (
+ not source_fenced
+ or not target_writable
+ or not _mysql_global_writer_switch_still_owned(
+ owner, committed_topology,
+ )
+ or _global_cluster_writer_cluster(gc) is not target
+ ):
+ raise RuntimeError("global topology changed after commit")
+ if not _finish_mysql_global_writer_switch(owner):
+ raise RuntimeError(
+ "global writer finalization could not be persisted"
+ )
+ result = _global_cluster_writer_switch_response(
+ gc,
+ current_writer,
+ target_member,
+ allow_data_loss=False,
+ )
+ finally:
+ _shared_container_lock.release()
+ return result
+ except Exception as e:
+ logger.warning("RDS: planned global switchover for %s failed: %s", gc_id, e)
+ if metadata_flipped:
+ _fence_or_contain_mysql_cluster(source)
+ _fence_or_contain_mysql_cluster(target)
+ with _shared_container_lock:
+ if _active_mysql_global_writer_switch(gc) is owner:
+ _finish_mysql_global_writer_switch(
+ owner, repair_required=True,
+ )
+ else:
+ _rollback_mysql_global_writer_switch(
+ owner,
+ target_channel_changed=target_channel_changed,
+ reverse_channel_attempted=reverse_channel_attempted,
+ )
+ return _not_ready_error()
+ finally:
+ _close_mysql_resources("switchover fence", connection=fence_conn)
+
+
+def _switch_global_cluster_writer(p, *, allow_data_loss=False):
+ with _shared_container_lock:
+ return _switch_global_cluster_writer_locked(
+ p, allow_data_loss=allow_data_loss,
+ )
+
+
+def _switch_global_cluster_writer_locked(p, *, allow_data_loss=False):
+ resolved, error = _resolve_global_writer_switch(p)
+ if error:
+ return error
+ gc, _target, target_member, current_writer = resolved
+ if _active_mysql_global_writer_switch(gc):
+ return _error(
+ "InvalidGlobalClusterStateFault",
+ f"Global cluster {gc['GlobalClusterIdentifier']} already has a "
+ "writer switchover in progress.",
+ 400,
+ )
+ _set_global_cluster_writer(gc, target_member)
+ gc["Status"] = "available"
+
+ return _global_cluster_writer_switch_response(
+ gc,
+ current_writer,
+ target_member,
+ allow_data_loss=allow_data_loss,
+ )
+
+
def _switchover_global_cluster(p):
- result = _switch_global_cluster_writer(p, allow_data_loss=False)
+ result = _planned_mysql_global_writer_switch(p)
if isinstance(result, tuple):
return result
return _xml(200, "SwitchoverGlobalClusterResponse",
@@ -7704,7 +8683,10 @@ def _failover_global_cluster(p):
"AllowDataLoss and Switchover cannot both be specified.",
400,
)
- result = _switch_global_cluster_writer(p, allow_data_loss=allow_data_loss)
+ if allow_data_loss:
+ result = _switch_global_cluster_writer(p, allow_data_loss=True)
+ else:
+ result = _planned_mysql_global_writer_switch(p)
if isinstance(result, tuple):
return result
return _xml(200, "FailoverGlobalClusterResponse",
diff --git a/tests/test_rds.py b/tests/test_rds.py
index 6a0c2520..291546a1 100644
--- a/tests/test_rds.py
+++ b/tests/test_rds.py
@@ -5052,6 +5052,8 @@ def test_rds_restore_state_respawns_docker_container(monkeypatch):
"pending-password-rotation",
"global-secondary",
"global-secondary-before-control-user",
+ "repair-required",
+ "repair-unfenced",
"last-member-deleted-during-readiness",
"last-members-deleted-before-start",
],
@@ -5072,6 +5074,7 @@ def test_rds_restore_state_respawns_one_container_per_cluster(
rotations = []
grants = []
replication_configs = []
+ forced_repair_removals = []
remaining_legacy_names = set()
legacy_owner_by_name = {}
writer_legacy_container_name = [None]
@@ -5222,6 +5225,22 @@ def test_rds_restore_state_respawns_one_container_per_cluster(
"_configure_or_defer_mysql_replication",
_configure_replication,
)
+ if scenario.startswith("repair-"):
+ monkeypatch.setattr(
+ m,
+ "_mysql_writer_fence_active",
+ lambda _cluster: scenario == "repair-required",
+ )
+ if scenario == "repair-unfenced":
+ monkeypatch.setattr(
+ m, "_stop_cluster_shared_container", lambda *_args: False,
+ )
+ monkeypatch.setattr(
+ m,
+ "_force_remove_cluster_compute_preserving_volume",
+ lambda cluster_id, _cluster: forced_repair_removals.append(cluster_id)
+ or True,
+ )
if scenario.startswith("global-secondary"):
monkeypatch.setattr(m, "_mysql_replication_secondary", lambda _cluster: True)
@@ -5280,6 +5299,15 @@ def test_rds_restore_state_respawns_one_container_per_cluster(
cluster_record["_mysql_gtid_initialized_at_creation"] = True
if scenario == "global-secondary":
cluster_record["_mysql_control_user_ready"] = True
+ elif scenario.startswith("repair-"):
+ cluster_record.update({
+ "DBClusterArn": (
+ "arn:aws:rds:us-east-1:111111111111:cluster:"
+ "restored-shared-cluster"
+ ),
+ "GlobalClusterIdentifier": "repair-global",
+ "_mysql_gtid_initialized_at_creation": True,
+ })
clusters.set_scoped(account_id, region, cluster_id, cluster_record)
instances = AccountRegionScopedDict()
for db_id in ("restored-writer", "restored-reader"):
@@ -5304,6 +5332,20 @@ def test_rds_restore_state_respawns_one_container_per_cluster(
m._instances.clear()
m._clusters.clear()
try:
+ global_clusters = {}
+ if scenario.startswith("repair-"):
+ global_clusters = {
+ "repair-global": {
+ "GlobalClusterIdentifier": "repair-global",
+ "GlobalClusterMembers": [{
+ "DBClusterArn": cluster_record["DBClusterArn"],
+ "IsWriter": True,
+ }],
+ m._MYSQL_GLOBAL_SWITCHOVER_STATE: {
+ "state": "repair_required",
+ },
+ },
+ }
m.restore_state({
"instances": instances,
"clusters": clusters,
@@ -5313,7 +5355,7 @@ def test_rds_restore_state_respawns_one_container_per_cluster(
"db_cluster_param_groups": {},
"db_cluster_snapshots": {},
"option_groups": {},
- "global_clusters": {},
+ "global_clusters": global_clusters,
"tags": {},
"port_counter": 16010,
})
@@ -5429,6 +5471,23 @@ def test_rds_restore_state_respawns_one_container_per_cluster(
assert removed_volumes == []
return
+ if scenario.startswith("repair-"):
+ restored_cluster = m._clusters.get(cluster_id)
+ assert {"--read-only=ON", "--super-read-only=ON"} <= set(
+ runs[0]["command"],
+ )
+ assert restored_cluster["_shared_container_ready"] is False
+ assert all(
+ m._instances.get(db_id)["DBInstanceStatus"] == "failed"
+ for db_id in ("restored-writer", "restored-reader")
+ )
+ assert replication_configs == []
+ assert removed_volumes == []
+ assert forced_repair_removals == (
+ [cluster_id] if scenario == "repair-unfenced" else []
+ )
+ return
+
assert len(runs) == 1
assert runs[0]["name"] == m._rds_cluster_docker_name(cluster_id)
if scenario.startswith("global-secondary"):
@@ -6568,17 +6627,10 @@ def _delete_global_cluster(client, global_id):
def _cleanup_two_member_global(east, global_id, primary_arn=None, secondary_arn=None):
- if primary_arn:
- try:
- east.switchover_global_cluster(
- GlobalClusterIdentifier=global_id,
- TargetDbClusterIdentifier=primary_arn,
- )
- except ClientError:
- pass
- for cluster_arn in (secondary_arn, primary_arn):
- if cluster_arn:
- _remove_global_member(east, global_id, cluster_arn)
+ for _attempt in range(2):
+ for cluster_arn in (secondary_arn, primary_arn):
+ if cluster_arn:
+ _remove_global_member(east, global_id, cluster_arn)
_delete_global_cluster(east, global_id)
@@ -7075,7 +7127,7 @@ def test_aurora_global_metadata_spans_regions():
_delete_cluster(east, primary_id)
-def test_switchover_global_cluster_promotes_foreign_region_member_arn():
+def test_switchover_global_cluster_rejects_headless_foreign_region_target():
east = _regional_rds("us-east-1")
west = _regional_rds("us-west-2")
suffix = uuid.uuid4().hex[:8]
@@ -7106,39 +7158,21 @@ def test_switchover_global_cluster_promotes_foreign_region_member_arn():
)["DBCluster"]
secondary_arn = secondary["DBClusterArn"]
- response = east.switchover_global_cluster(
- GlobalClusterIdentifier=global_id,
- TargetDbClusterIdentifier=secondary_arn,
- )["GlobalCluster"]
- assert response["Status"] == "switching-over"
- assert response["FailoverState"]["Status"] == "pending"
- assert response["FailoverState"]["FromDbClusterArn"] == primary_arn
- assert response["FailoverState"]["ToDbClusterArn"] == secondary_arn
- assert response["FailoverState"]["IsDataLossAllowed"] is False
-
- members = {m["DBClusterArn"]: m for m in response["GlobalClusterMembers"]}
- assert members[primary_arn]["IsWriter"] is False
- assert members[secondary_arn]["IsWriter"] is True
- assert members[secondary_arn]["Readers"] == [primary_arn]
+ with pytest.raises(ClientError) as exc:
+ east.switchover_global_cluster(
+ GlobalClusterIdentifier=global_id,
+ TargetDbClusterIdentifier=secondary_arn,
+ )
+ assert exc.value.response["Error"]["Code"] == "InvalidDBClusterStateFault"
- final = west.describe_global_clusters(
+ final = east.describe_global_clusters(
GlobalClusterIdentifier=global_id,
)["GlobalClusters"][0]
final_members = {m["DBClusterArn"]: m for m in final["GlobalClusterMembers"]}
assert final["Status"] == "available"
assert "FailoverState" not in final
- assert final_members[primary_arn]["IsWriter"] is False
- assert final_members[secondary_arn]["IsWriter"] is True
-
- switchback = west.switchover_global_cluster(
- GlobalClusterIdentifier=global_id,
- TargetDbClusterIdentifier=primary_arn,
- )["GlobalCluster"]
- switchback_members = {
- m["DBClusterArn"]: m for m in switchback["GlobalClusterMembers"]
- }
- assert switchback_members[primary_arn]["IsWriter"] is True
- assert switchback_members[secondary_arn]["IsWriter"] is False
+ assert final_members[primary_arn]["IsWriter"] is True
+ assert final_members[secondary_arn]["IsWriter"] is False
finally:
_cleanup_two_member_global(east, global_id, primary_arn, secondary_arn)
_delete_cluster(west, secondary_id)
@@ -10530,6 +10564,182 @@ def test_rds_mysql_quiescence_epoch_change_suppresses_result(monkeypatch):
assert m._wait_for_mysql_writer_quiescence(cluster, 1) is None
+def test_rds_mysql_writer_switch_state_round_trip_requires_repair():
+ from ministack.services import rds as m
+
+ gc_id = "persisted-switch"
+ global_cluster = {
+ "GlobalClusterIdentifier": gc_id,
+ "GlobalClusterMembers": [],
+ m._MYSQL_GLOBAL_SWITCHOVER_STATE: {
+ "state": "in_progress",
+ "source_arn": "source-arn",
+ "target_arn": "target-arn",
+ },
+ }
+ m._global_clusters.clear()
+ m._global_clusters[gc_id] = global_cluster
+ try:
+ persisted = m.get_state()
+ m._global_clusters.clear()
+
+ m.restore_state({
+ "instances": {},
+ "clusters": {},
+ "global_clusters": persisted["global_clusters"],
+ })
+
+ restored = m._global_clusters[gc_id]
+ marker = restored[m._MYSQL_GLOBAL_SWITCHOVER_STATE]
+ assert marker == {
+ "state": "repair_required",
+ "source_arn": "source-arn",
+ "target_arn": "target-arn",
+ }
+ assert m._active_mysql_global_writer_switch(restored) is marker
+ finally:
+ m._global_clusters.clear()
+
+
+def test_rds_repair_containment_does_not_treat_docker_error_as_absent(
+ monkeypatch,
+):
+ from ministack.services import rds as m
+
+ class FailingContainers:
+ def get(self, _identifier):
+ raise TimeoutError("Docker daemon timed out")
+
+ class FakeDocker:
+ def __init__(self):
+ self.containers = FailingContainers()
+
+ cluster = {
+ "DBClusterArn": (
+ "arn:aws:rds:us-west-2:111111111111:cluster:repair-cluster"
+ ),
+ "_shared_container_id": "possibly-running-container",
+ }
+ monkeypatch.setattr(m, "_get_docker", lambda: FakeDocker())
+
+ assert m._force_remove_cluster_compute_preserving_volume(
+ "repair-cluster", cluster,
+ ) is False
+ assert cluster["_shared_container_id"] == "possibly-running-container"
+
+
+def test_rds_repair_containment_uses_cluster_arn_scope(monkeypatch):
+ from ministack.services import rds as m
+
+ removed = []
+
+ class NotFound(Exception):
+ status_code = 404
+
+ class Container:
+ id = "target-container"
+ labels = {
+ "ministack": "rds",
+ "cluster_id": "repair-cluster",
+ "account_id": "111111111111",
+ "region": "us-west-2",
+ }
+
+ def remove(self, **kwargs):
+ removed.append(kwargs)
+
+ container = Container()
+ target_name = m._rds_cluster_docker_name(
+ "repair-cluster", "111111111111", "us-west-2",
+ )
+
+ class Containers:
+ def get(self, identifier):
+ if not removed and identifier in (container.id, target_name):
+ return container
+ raise NotFound()
+
+ class FakeDocker:
+ def __init__(self):
+ self.containers = Containers()
+
+ cluster = {
+ "DBClusterArn": (
+ "arn:aws:rds:us-west-2:111111111111:cluster:repair-cluster"
+ ),
+ "_shared_container_id": container.id,
+ "_shared_internal_address": "172.20.0.20",
+ "_shared_internal_port": 3306,
+ }
+ monkeypatch.setattr(m, "_get_docker", lambda: FakeDocker())
+
+ assert m._contain_cluster_compute_preserving_volume(
+ "repair-cluster", cluster,
+ ) is True
+ assert removed == [{"force": True, "v": False}]
+ assert cluster["_shared_container_id"] is None
+ assert cluster["_shared_internal_address"] is None
+ assert cluster["_shared_internal_port"] is None
+
+
+def test_rds_member_status_update_uses_cluster_arn_scope():
+ from ministack.services import rds as m
+
+ instance_id = "same-name-member"
+ east = {"DBInstanceIdentifier": instance_id, "DBInstanceStatus": "available"}
+ west = {"DBInstanceIdentifier": instance_id, "DBInstanceStatus": "available"}
+ m._instances.clear()
+ try:
+ m._instances.set_scoped(
+ "111111111111", "us-east-1", instance_id, east,
+ )
+ m._instances.set_scoped(
+ "111111111111", "us-west-2", instance_id, west,
+ )
+ cluster = {
+ "DBClusterArn": (
+ "arn:aws:rds:us-west-2:111111111111:cluster:repair-cluster"
+ ),
+ "DBClusterMembers": [{"DBInstanceIdentifier": instance_id}],
+ }
+
+ m._set_cluster_members_status(cluster, "failed")
+
+ assert east["DBInstanceStatus"] == "available"
+ assert west["DBInstanceStatus"] == "failed"
+ finally:
+ m._instances.clear()
+
+
+def test_rds_cluster_refence_uses_one_total_deadline(monkeypatch):
+ from ministack.services import rds as m
+
+ class Connection:
+ def close(self):
+ pass
+
+ deadlines = []
+ monkeypatch.setattr(m.time, "monotonic", lambda: 100.0)
+ monkeypatch.setattr(
+ m,
+ "_mysql_replication_connection",
+ lambda _cluster, timeout: Connection() if timeout == 3 else None,
+ )
+ monkeypatch.setattr(
+ m,
+ "_set_mysql_writer_fence",
+ lambda _conn, *, deadline: deadlines.append(deadline) or True,
+ )
+ monkeypatch.setattr(
+ m,
+ "_mysql_writer_fence_active",
+ lambda _cluster, *, deadline: deadlines.append(deadline) or True,
+ )
+
+ assert m._set_mysql_cluster_fenced({}, timeout=3) is True
+ assert deadlines == [103.0, 103.0]
+
+
def _mysql_switchover_unit_topology(monkeypatch, m):
topology = _mysql_replication_unit_topology()
writer, target, writer_member, target_member, global_cluster = topology
@@ -10569,6 +10779,9 @@ def _mysql_switchover_unit_topology(monkeypatch, m):
}] if cluster.get("DBClusterMembers") else [],
)
monkeypatch.setattr(m, "_mysql_global_writer_switch_owners", {})
+ monkeypatch.setattr(
+ m, "_wait_for_mysql_replication_healthy", lambda *_a, **_k: True,
+ )
return topology
@@ -10579,106 +10792,1072 @@ def _mysql_switchover_params(target="secondary"):
}
-def test_rds_mysql_gtid_wait_recomputes_shared_deadline(monkeypatch):
+def test_rds_mysql_switchover_claim_is_durable_when_persistence_enabled(
+ monkeypatch,
+ tmp_path,
+):
from ministack.services import rds as m
- now = [100.0]
- observed = {}
- monkeypatch.setattr(m.time, "monotonic", lambda: now[0])
-
- class Cursor:
- def __init__(self, connection):
- self.connection = connection
-
- def execute(self, statement, params):
- observed["sql"] = (statement, params)
-
- def fetchone(self):
- return (0,)
+ writer, target, _writer_member, target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+ m._global_clusters.clear()
+ m._global_clusters["global-repl"] = global_cluster
+ monkeypatch.setattr(m.persistence, "PERSIST_STATE", True)
+ monkeypatch.setattr(m.persistence, "STATE_DIR", str(tmp_path))
+ try:
+ owner = m._claim_mysql_global_writer_switch(
+ global_cluster, writer, target,
+ )
- def close(self):
- pass
+ assert owner is not None
+ marker = global_cluster[m._MYSQL_GLOBAL_SWITCHOVER_STATE]
+ assert marker["operation_id"]
+ persisted = m.persistence.load_state("rds")
+ assert m._persisted_mysql_global_writer_switch_marker(
+ persisted, global_cluster,
+ ) == marker
+
+ m._set_global_cluster_writer(global_cluster, target_member)
+ assert m._finish_mysql_global_writer_switch(owner) is True
+ persisted = m.persistence.load_state("rds")
+ persisted_gc = m._persisted_mysql_global_writer_switch_cluster(
+ persisted, global_cluster,
+ )
+ assert persisted_gc is not None
+ assert m._MYSQL_GLOBAL_SWITCHOVER_STATE not in persisted_gc
+ assert next(
+ member
+ for member in persisted_gc["GlobalClusterMembers"]
+ if member["DBClusterArn"] == target["DBClusterArn"]
+ )["IsWriter"] is True
+ finally:
+ m._mysql_global_writer_switch_owners.clear()
+ m._global_clusters.clear()
- class Connection:
- def cursor(self):
- now[0] += 4
- return Cursor(self)
- def close(self):
- pass
+def test_rds_mysql_switchover_claim_fails_closed_when_persistence_fails(
+ monkeypatch,
+):
+ from ministack.services import rds as m
- def connect(_cluster, *, timeout):
- observed["connect_timeout"] = timeout
- return Connection()
+ writer, target, _writer_member, _target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+ monkeypatch.setattr(m.persistence, "PERSIST_STATE", True)
+ monkeypatch.setattr(m.persistence, "save_state", lambda *_args: None)
+ monkeypatch.setattr(m.persistence, "load_state", lambda *_args: None)
- monkeypatch.setattr(m, "_mysql_replication_connection", connect)
+ owner = m._claim_mysql_global_writer_switch(global_cluster, writer, target)
- assert m._wait_for_mysql_gtid(
- {"DBClusterIdentifier": "secondary"}, "source:1-9", 30,
- deadline=130.0,
- ) is True
- assert observed["connect_timeout"] == 30.0
- assert observed["sql"][1] == ("source:1-9", 26.0)
+ assert owner is None
+ assert m._MYSQL_GLOBAL_SWITCHOVER_STATE not in global_cluster
+ assert m._mysql_global_writer_switch_owners == {}
[email protected]("mutation", ["password", "topology"])
-def test_rds_active_switchover_rejects_mutation(monkeypatch, mutation):
+def test_rds_mysql_switchover_finalization_failure_retains_repair_state(
+ monkeypatch,
+):
from ministack.services import rds as m
- writer, target, writer_member, target_member, global_cluster = (
+ writer, target, _writer_member, _target_member, global_cluster = (
_mysql_switchover_unit_topology(monkeypatch, m)
)
- with m._shared_container_lock:
- owner = m._claim_mysql_global_writer_switch(global_cluster, writer, target)
- if mutation == "password":
- result = m._modify_db_cluster({
- "DBClusterIdentifier": "primary",
- "MasterUserPassword": "changed-password",
- })
- elif mutation == "topology":
- result = m._remove_from_global_cluster({
- "GlobalClusterIdentifier": "global-repl",
- "DbClusterIdentifier": "secondary",
- })
- else:
- result = m._failover_global_cluster({
- "GlobalClusterIdentifier": "global-repl",
- "TargetDbClusterIdentifier": "secondary",
- "AllowDataLoss": "true",
- })
+ persist_results = iter((True, False))
+ monkeypatch.setattr(
+ m,
+ "_durably_persist_mysql_global_writer_switch_state",
+ lambda _gc, _marker: next(persist_results),
+ )
- assert result[0] == 400
+ owner = m._claim_mysql_global_writer_switch(global_cluster, writer, target)
+
+ assert owner is not None
+ assert m._finish_mysql_global_writer_switch(owner) is False
assert m._active_mysql_global_writer_switch(global_cluster) is owner
- assert writer_member["IsWriter"] is True
- assert target_member["IsWriter"] is False
- assert writer.get("_MasterUserPassword") != "changed-password"
- assert target["GlobalClusterIdentifier"] == "global-repl"
- assert target["_mysql_replication_source_arn"] == writer["DBClusterArn"]
+ assert global_cluster[m._MYSQL_GLOBAL_SWITCHOVER_STATE] == {
+ "state": "repair_required",
+ "source_arn": writer["DBClusterArn"],
+ "target_arn": target["DBClusterArn"],
+ "operation_id": owner["operation_id"],
+ }
@pytest.mark.parametrize(
- ("handler_name", "impl_name"),
+ "invalid",
[
- ("_create_db_instance", "_create_db_instance_impl"),
- ("_delete_db_instance", "_delete_db_instance_impl"),
+ "current-writer", "third-member", "engine", "version", "storage",
+ "headless", "unready", "status", "member",
],
)
-def test_rds_instance_mutation_prevents_switchover_claim(
+def test_rds_switchover_requires_ready_two_member_mysql8_topology(
monkeypatch,
- handler_name,
- impl_name,
+ invalid,
):
from ministack.services import rds as m
- writer, target, _writer_member, _target_member, global_cluster = (
+ writer, target, writer_member, target_member, global_cluster = (
_mysql_switchover_unit_topology(monkeypatch, m)
)
- entered = threading.Event()
- release = threading.Event()
- claimed = threading.Event()
- order = []
+ target_id = "secondary"
+ if invalid == "current-writer":
+ writer_member["IsWriter"] = False
+ target_member["IsWriter"] = True
+ target_id = "secondary"
+ elif invalid == "third-member":
+ global_cluster["GlobalClusterMembers"].append({
+ "DBClusterArn": "arn:aws:rds:eu-west-1:111111111111:cluster:third",
+ "IsWriter": False,
+ })
+ elif invalid == "engine":
+ target["Engine"] = "aurora-postgresql"
+ elif invalid == "version":
+ target["EngineVersion"] = "8.0.mysql_aurora.3.11.1"
+ elif invalid == "storage":
+ target["_shared_storage_initialized"] = False
+ elif invalid == "headless":
+ target["DBClusterMembers"] = []
+ elif invalid == "unready":
+ target["_shared_container_ready"] = False
+ elif invalid == "status":
+ target["Status"] = "creating"
+ else:
+ monkeypatch.setattr(
+ m, "_cluster_member_instances",
+ lambda cluster: [{"DBInstanceStatus": "stopped"}]
+ if cluster is target else [{"DBInstanceStatus": "available"}],
+ )
- def mutation_impl(_params):
+ monkeypatch.setattr(
+ m, "_ensure_mysql_control_user",
+ lambda *_args, **_kwargs: pytest.fail("invalid topology reached MySQL"),
+ )
+ result = m._planned_mysql_global_writer_switch(
+ _mysql_switchover_params(target_id),
+ )
+ assert result[0] == 400
+ assert writer_member["IsWriter"] is (invalid != "current-writer")
+ assert target_member["IsWriter"] is (invalid == "current-writer")
+
+
+def test_rds_switchover_preserves_non_mysql_multi_member_metadata_behavior(
+ monkeypatch,
+):
+ from ministack.services import rds as m
+
+ writer, target, writer_member, target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+ writer["Engine"] = target["Engine"] = "aurora-postgresql"
+ global_cluster["Engine"] = "aurora-postgresql"
+ global_cluster["GlobalClusterMembers"].append({
+ "DBClusterArn": "arn:aws:rds:eu-west-1:111111111111:cluster:third",
+ "IsWriter": False,
+ })
+
+ result = m._planned_mysql_global_writer_switch(_mysql_switchover_params())
+
+ assert result["Status"] == "switching-over"
+ assert writer_member["IsWriter"] is False
+ assert target_member["IsWriter"] is True
+
+
+ "mode",
+ [{}, {"Switchover": "true"}, {"AllowDataLoss": "false"}],
+)
+def test_rds_failover_lossless_modes_use_planned_switchover(monkeypatch, mode):
+ from ministack.services import rds as m
+
+ *_, global_cluster = _mysql_switchover_unit_topology(monkeypatch, m)
+ calls = []
+ monkeypatch.setattr(
+ m,
+ "_planned_mysql_global_writer_switch",
+ lambda params: calls.append(params) or global_cluster,
+ )
+ monkeypatch.setattr(
+ m,
+ "_switch_global_cluster_writer",
+ lambda *_args, **_kwargs: pytest.fail("lossless failover used metadata path"),
+ )
+ params = _mysql_switchover_params() | mode
+
+ status, _headers, _body = m._failover_global_cluster(params)
+
+ assert status == 200
+ assert calls == [params]
+
+
+def test_rds_switchover_orders_lossless_data_plane_before_metadata(monkeypatch):
+ from ministack.services import rds as m
+
+ writer, target, writer_member, target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+ events = []
+
+ class FenceConnection:
+ def close(self):
+ events.append("fence-close")
+
+ monkeypatch.setattr(
+ m, "_ensure_mysql_control_user",
+ lambda cluster, **kwargs: events.append(
+ ("control", cluster, kwargs.get("require_quiescence_grants", False)),
+ ) or True,
+ )
+ monkeypatch.setattr(
+ m, "_ensure_mysql_replication_user",
+ lambda cluster, **_kwargs: events.append(("replication-user", cluster)) or True,
+ )
+ monkeypatch.setattr(
+ m, "_mysql_writer_fence_active",
+ lambda cluster, **_kwargs: events.append(("target-fenced", cluster)) or True,
+ )
+ monkeypatch.setattr(
+ m,
+ "_durably_persist_mysql_global_writer_switch_state",
+ lambda _gc, marker: events.append(
+ "persist-claim" if marker else "persist-final",
+ ) or True,
+ )
+ monkeypatch.setattr(
+ m, "_mysql_replication_connection",
+ lambda cluster, **_kwargs: events.append(("fence-connect", cluster))
+ or FenceConnection(),
+ )
+ monkeypatch.setattr(
+ m, "_set_mysql_writer_fence",
+ lambda _conn, **_kwargs: events.append("source-fence") or True,
+ )
+ monkeypatch.setattr(
+ m, "_wait_for_mysql_writer_quiescence",
+ lambda cluster, _timeout, **_kwargs: events.append(("drain", cluster)) or "source:1-9",
+ )
+ monkeypatch.setattr(
+ m, "_wait_for_mysql_gtid",
+ lambda cluster, gtid, _timeout, **_kwargs: events.append(("gtid", cluster, gtid)) or True,
+ )
+ monkeypatch.setattr(
+ m, "_reset_mysql_replication_channel",
+ lambda cluster_id, cluster, **kwargs: events.append(
+ ("reset", cluster_id, cluster, kwargs["clear_super_read_only"]),
+ ) or True,
+ )
+
+ def configure(cluster_id, replica, source, **_kwargs):
+ assert writer_member["IsWriter"] is True
+ assert target_member["IsWriter"] is False
+ events.append(("configure", cluster_id, replica, source))
+ replica["_mysql_replication_source_arn"] = source["DBClusterArn"]
+ return True
+
+ def healthy(replica, source, **_kwargs):
+ events.append(("healthy", replica, source))
+ return True
+
+ def writable(cluster, **_kwargs):
+ assert writer_member["IsWriter"] is False
+ assert target_member["IsWriter"] is True
+ events.append(("writable", cluster))
+ return True
+
+ monkeypatch.setattr(m, "_configure_mysql_replica_from_source", configure)
+ monkeypatch.setattr(m, "_wait_for_mysql_replication_healthy", healthy)
+ monkeypatch.setattr(m, "_set_mysql_cluster_writable", writable)
+ monkeypatch.setattr(
+ m, "_mysql_control_query",
+ lambda cluster, *_args, **_kwargs: ((0, 0),) if cluster is target else None,
+ )
+ monkeypatch.setattr(m, "_sync_global_mysql_credentials", lambda *_args: None)
+ response_lock_held = []
+ original_response = m._global_cluster_writer_switch_response
+
+ def response_under_lock(*args, **kwargs):
+ acquired = []
+
+ def try_lock():
+ locked = m._shared_container_lock.acquire(blocking=False)
+ acquired.append(locked)
+ if locked:
+ m._shared_container_lock.release()
+
+ contender = threading.Thread(target=try_lock)
+ contender.start()
+ contender.join(timeout=1)
+ assert not contender.is_alive()
+ response_lock_held.append(not acquired[0])
+ return original_response(*args, **kwargs)
+
+ monkeypatch.setattr(
+ m, "_global_cluster_writer_switch_response", response_under_lock,
+ )
+
+ status, _headers, body = m._switchover_global_cluster(
+ _mysql_switchover_params(),
+ )
+ body = body.decode()
+
+ assert status == 200
+ assert "<Status>switching-over</Status>" in body
+ assert f"<FromDbClusterArn>{writer['DBClusterArn']}</FromDbClusterArn>" in body
+ assert f"<ToDbClusterArn>{target['DBClusterArn']}</ToDbClusterArn>" in body
+ assert "<IsDataLossAllowed>false</IsDataLossAllowed>" in body
+ assert global_cluster["Status"] == "available"
+ assert writer_member["IsWriter"] is False
+ assert target_member["IsWriter"] is True
+ assert target_member["Readers"] == [writer["DBClusterArn"]]
+ assert m._mysql_global_writer_switch_owners == {}
+ assert m._MYSQL_GLOBAL_SWITCHOVER_STATE not in global_cluster
+ assert response_lock_held == [True]
+ assert [event[0] if isinstance(event, tuple) else event for event in events] == [
+ "control", "control", "replication-user", "target-fenced",
+ "persist-claim", "fence-connect", "source-fence", "drain",
+ "fence-close", "gtid", "reset", "configure", "healthy", "writable",
+ "target-fenced", "persist-final",
+ ]
+ assert ("reset", "secondary", target, False) in events
+ assert ("configure", "primary", writer, target) in events
+
+
[email protected]("committed", [False, True])
+def test_rds_switchover_retry_recovers_restored_repair_marker(
+ monkeypatch,
+ committed,
+):
+ from ministack.services import rds as m
+
+ writer, target, writer_member, target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+ operation_id = "interrupted-operation"
+ if committed:
+ m._set_global_cluster_writer(global_cluster, target_member)
+ writer["_mysql_replication_source_arn"] = target["DBClusterArn"]
+ target.pop("_mysql_replication_source_arn", None)
+ global_cluster[m._MYSQL_GLOBAL_SWITCHOVER_STATE] = {
+ "state": "repair_required",
+ "source_arn": writer["DBClusterArn"],
+ "target_arn": target["DBClusterArn"],
+ "operation_id": operation_id,
+ }
+ for cluster in (writer, target):
+ cluster["_shared_container_ready"] = False
+ cluster["Status"] = "creating"
+
+ class FenceConnection:
+ def close(self):
+ pass
+
+ monkeypatch.setattr(m, "_ensure_mysql_control_user", lambda *_a, **_k: True)
+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)
+ monkeypatch.setattr(m, "_mysql_writer_fence_active", lambda *_a, **_k: True)
+ monkeypatch.setattr(
+ m, "_mysql_replication_connection", lambda *_a, **_k: FenceConnection(),
+ )
+ monkeypatch.setattr(m, "_set_mysql_writer_fence", lambda *_a, **_k: True)
+ monkeypatch.setattr(
+ m, "_wait_for_mysql_writer_quiescence",
+ lambda *_a, **_k: "source:1-9",
+ )
+ monkeypatch.setattr(m, "_wait_for_mysql_gtid", lambda *_a, **_k: True)
+
+ reset_calls = []
+ monkeypatch.setattr(
+ m,
+ "_reset_mysql_replication_channel",
+ lambda *_a, **kwargs: reset_calls.append(kwargs) or True,
+ )
+
+ def configure(_cluster_id, replica, source, **_kwargs):
+ replica["_mysql_replication_source_arn"] = source["DBClusterArn"]
+ return True
+
+ monkeypatch.setattr(m, "_configure_mysql_replica_from_source", configure)
+ monkeypatch.setattr(m, "_wait_for_mysql_replication_healthy", lambda *_a, **_k: True)
+ monkeypatch.setattr(m, "_set_mysql_cluster_writable", lambda *_a, **_k: True)
+ monkeypatch.setattr(m, "_mysql_control_query", lambda *_a, **_k: ((0, 0),))
+ monkeypatch.setattr(m, "_sync_global_mysql_credentials", lambda *_a: None)
+ monkeypatch.setattr(
+ m,
+ "_durably_persist_mysql_global_writer_switch_state",
+ lambda _gc, expected: expected is None,
+ )
+
+ result = m._planned_mysql_global_writer_switch(_mysql_switchover_params())
+
+ assert isinstance(result, dict)
+ assert writer_member["IsWriter"] is False
+ assert target_member["IsWriter"] is True
+ assert m._MYSQL_GLOBAL_SWITCHOVER_STATE not in global_cluster
+ assert m._mysql_global_writer_switch_owners == {}
+ if committed:
+ assert reset_calls == []
+ else:
+ assert reset_calls and reset_calls[0]["allow_missing_channel"] is True
+
+
+ "failure",
+ ["drain", "gtid", "target-reset", "reverse", "reverse-health", "commit-gate"],
+)
+def test_rds_switchover_failure_restores_original_topology(monkeypatch, failure):
+ from ministack.services import rds as m
+
+ writer, target, writer_member, target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+ events = []
+
+ class FenceConnection:
+ def close(self):
+ events.append("fence-close")
+
+ monkeypatch.setattr(m, "_ensure_mysql_control_user", lambda *_a, **_k: True)
+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)
+ monkeypatch.setattr(m, "_mysql_writer_fence_active", lambda _cluster, **_kwargs: True)
+ monkeypatch.setattr(m, "_mysql_replication_connection", lambda *_a, **_k: FenceConnection())
+ monkeypatch.setattr(m, "_set_mysql_writer_fence", lambda _conn, **_kwargs: True)
+ monkeypatch.setattr(
+ m, "_wait_for_mysql_writer_quiescence",
+ lambda *_args, **_kwargs: None if failure == "drain" else "source:1-9",
+ )
+ monkeypatch.setattr(
+ m, "_wait_for_mysql_gtid",
+ lambda *_args, **_kwargs: failure != "gtid",
+ )
+
+ def reset(cluster_id, cluster, **kwargs):
+ events.append((
+ "reset", cluster_id, cluster,
+ kwargs.get("allow_missing_channel", False),
+ ))
+ if failure == "target-reset" and cluster is target:
+ raise RuntimeError("target reset failed")
+ return True
+
+ def configure(cluster_id, replica, source, **_kwargs):
+ events.append(("configure", cluster_id, replica, source))
+ if failure == "reverse" and replica is writer and source is target:
+ return False
+ if failure == "commit-gate" and replica is writer:
+ target["_shared_container_ready"] = False
+ elif failure == "commit-gate" and replica is target:
+ target["_shared_container_ready"] = True
+ replica["_mysql_replication_source_arn"] = source["DBClusterArn"]
+ return True
+
+ monkeypatch.setattr(m, "_reset_mysql_replication_channel", reset)
+ monkeypatch.setattr(m, "_configure_mysql_replica_from_source", configure)
+
+ def healthy(replica, source, **_kwargs):
+ events.append(("healthy", replica, source))
+ return not (failure == "reverse-health" and replica is writer)
+
+ monkeypatch.setattr(m, "_wait_for_mysql_replication_healthy", healthy)
+ monkeypatch.setattr(
+ m, "_set_mysql_cluster_writable",
+ lambda cluster, **_kwargs: events.append(("writable", cluster)) or True,
+ )
+
+ result = m._planned_mysql_global_writer_switch(_mysql_switchover_params())
+
+ assert result[0] == 400
+ assert writer_member["IsWriter"] is True
+ assert target_member["IsWriter"] is False
+ assert global_cluster["Status"] == "available"
+ assert m._mysql_global_writer_switch_owners == {}
+ assert [event for event in events if isinstance(event, tuple)][-1] == (
+ "writable", writer,
+ )
+ if failure in ("target-reset", "reverse", "reverse-health", "commit-gate"):
+ assert ("configure", "secondary", target, writer) in events
+ if failure in ("reverse", "reverse-health", "commit-gate"):
+ configure_index = events.index(("configure", "secondary", target, writer))
+ assert events[configure_index - 1] == ("reset", "primary", writer, True)
+ if failure == "reverse-health":
+ assert [event for event in events if event[0] == "healthy"] == [
+ ("healthy", writer, target),
+ ("healthy", target, writer),
+ ]
+
+
+ "failure", ["write-enable", "source-fence", "persist-final"],
+)
+def test_rds_switchover_post_commit_failure_secures_both_members(monkeypatch, failure):
+ from ministack.services import rds as m
+
+ writer, target, writer_member, target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+
+ class FenceConnection:
+ def close(self):
+ pass
+
+ monkeypatch.setattr(m, "_ensure_mysql_control_user", lambda *_a, **_k: True)
+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)
+ monkeypatch.setattr(
+ m,
+ "_mysql_writer_fence_active",
+ lambda cluster, **_kwargs: not (
+ failure == "source-fence" and cluster is writer
+ ),
+ )
+ monkeypatch.setattr(m, "_mysql_replication_connection", lambda *_a, **_k: FenceConnection())
+ monkeypatch.setattr(m, "_set_mysql_writer_fence", lambda _conn, **_kwargs: True)
+ monkeypatch.setattr(m, "_wait_for_mysql_writer_quiescence", lambda *_a, **_k: "source:1-9")
+ monkeypatch.setattr(m, "_wait_for_mysql_gtid", lambda *_a, **_k: True)
+ monkeypatch.setattr(m, "_reset_mysql_replication_channel", lambda *_a, **_k: True)
+ monkeypatch.setattr(
+ m, "_configure_mysql_replica_from_source", lambda *_a, **_k: True,
+ )
+ write_attempts = []
+ monkeypatch.setattr(
+ m, "_set_mysql_cluster_writable",
+ lambda cluster, **_kwargs: write_attempts.append(cluster)
+ or failure != "write-enable",
+ )
+ secured = []
+ monkeypatch.setattr(
+ m,
+ "_fence_or_contain_mysql_cluster",
+ lambda cluster: secured.append(cluster) or "fenced",
+ )
+ monkeypatch.setattr(m, "_sync_global_mysql_credentials", lambda *_args: None)
+ monkeypatch.setattr(
+ m,
+ "_durably_persist_mysql_global_writer_switch_state",
+ lambda _gc, marker: not (failure == "persist-final" and marker is None),
+ )
+
+ result = m._planned_mysql_global_writer_switch(_mysql_switchover_params())
+
+ assert result[0] == 400
+ assert writer_member["IsWriter"] is False
+ assert target_member["IsWriter"] is True
+ assert write_attempts == [target]
+ assert secured == [writer, target]
+ owner = m._active_mysql_global_writer_switch(global_cluster)
+ assert owner is not None and owner["state"] == "repair_required"
+ assert global_cluster[m._MYSQL_GLOBAL_SWITCHOVER_STATE]["state"] == (
+ "repair_required"
+ )
+ assert global_cluster["Status"] == "available"
+ assert b"<Status>switching-over</Status>" in m._describe_global_clusters({
+ "GlobalClusterIdentifier": "global-repl",
+ })[2]
+
+
+def test_rds_switchover_sync_failure_keeps_committed_topology_fenced(monkeypatch):
+ from ministack.services import rds as m
+
+ writer, target, writer_member, target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+ events = []
+
+ class FenceConnection:
+ def close(self):
+ pass
+
+ monkeypatch.setattr(m, "_ensure_mysql_control_user", lambda *_a, **_k: True)
+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)
+ monkeypatch.setattr(m, "_mysql_writer_fence_active", lambda *_a, **_k: True)
+ monkeypatch.setattr(
+ m, "_mysql_replication_connection", lambda *_a, **_k: FenceConnection(),
+ )
+ monkeypatch.setattr(m, "_set_mysql_writer_fence", lambda *_a, **_k: True)
+ monkeypatch.setattr(
+ m, "_wait_for_mysql_writer_quiescence", lambda *_a, **_k: "source:1-9",
+ )
+ monkeypatch.setattr(m, "_wait_for_mysql_gtid", lambda *_a, **_k: True)
+ monkeypatch.setattr(
+ m,
+ "_reset_mysql_replication_channel",
+ lambda _cluster_id, cluster, **_kwargs: events.append(("reset", cluster))
+ or True,
+ )
+ monkeypatch.setattr(
+ m,
+ "_configure_mysql_replica_from_source",
+ lambda _cluster_id, replica, source, **_kwargs: events.append(
+ ("configure", replica, source),
+ ) or True,
+ )
+ monkeypatch.setattr(
+ m,
+ "_set_mysql_cluster_writable",
+ lambda cluster, **_kwargs: events.append(("writable", cluster)) or True,
+ )
+
+ def fail_sync(*_args):
+ raise RuntimeError("credential sync failed")
+
+ monkeypatch.setattr(m, "_sync_global_mysql_credentials", fail_sync)
+
+ result = m._planned_mysql_global_writer_switch(_mysql_switchover_params())
+
+ assert result[0] == 400
+ assert writer_member["IsWriter"] is False
+ assert target_member["IsWriter"] is True
+ assert events == [("reset", target), ("configure", writer, target)]
+ owner = m._active_mysql_global_writer_switch(global_cluster)
+ assert owner is not None and owner["state"] == "repair_required"
+ assert global_cluster[m._MYSQL_GLOBAL_SWITCHOVER_STATE]["state"] == (
+ "repair_required"
+ )
+ assert global_cluster["Status"] == "available"
+ assert b"<Status>switching-over</Status>" in m._describe_global_clusters({
+ "GlobalClusterIdentifier": "global-repl",
+ })[2]
+
+
+ "rollback_failure",
+ [
+ "source-reset", "target-restore", "target-health", "target-fence",
+ "source-writable", "final-persist",
+ ],
+)
+def test_rds_switchover_failed_compensation_retains_owner(
+ monkeypatch,
+ rollback_failure,
+):
+ from ministack.services import rds as m
+
+ writer, target, writer_member, target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+
+ class FenceConnection:
+ def close(self):
+ pass
+
+ monkeypatch.setattr(m, "_ensure_mysql_control_user", lambda *_a, **_k: True)
+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)
+ monkeypatch.setattr(m, "_mysql_writer_fence_active", lambda *_a, **_k: True)
+ monkeypatch.setattr(m, "_mysql_replication_connection", lambda *_a, **_k: FenceConnection())
+ monkeypatch.setattr(m, "_set_mysql_writer_fence", lambda _conn, **_kwargs: True)
+ monkeypatch.setattr(m, "_wait_for_mysql_writer_quiescence", lambda *_a, **_k: "source:1-9")
+ monkeypatch.setattr(m, "_wait_for_mysql_gtid", lambda *_a, **_k: True)
+ def reset(_cluster_id, cluster, **_kwargs):
+ if rollback_failure == "source-reset" and cluster is writer:
+ raise RuntimeError("source reset failed")
+ return True
+
+ monkeypatch.setattr(m, "_reset_mysql_replication_channel", reset)
+
+ configured = []
+ def configure(_cluster_id, replica, _source, **_kwargs):
+ configured.append(replica)
+ if replica is writer:
+ return False
+ return rollback_failure != "target-restore"
+
+ monkeypatch.setattr(m, "_configure_mysql_replica_from_source", configure)
+ monkeypatch.setattr(
+ m,
+ "_wait_for_mysql_replication_healthy",
+ lambda replica, _source, **_kwargs: not (
+ rollback_failure == "target-health" and replica is target
+ ),
+ )
+ writable = []
+ monkeypatch.setattr(
+ m, "_set_mysql_cluster_writable",
+ lambda cluster, **_kwargs: writable.append(cluster)
+ or rollback_failure != "source-writable",
+ )
+ target_safety = []
+ if rollback_failure in ("target-fence", "final-persist"):
+ monkeypatch.setattr(
+ m,
+ "_fence_or_contain_mysql_cluster",
+ lambda cluster: target_safety.append(cluster) or (
+ "contained" if rollback_failure == "target-fence" else "fenced"
+ ),
+ )
+ monkeypatch.setattr(
+ m,
+ "_durably_persist_mysql_global_writer_switch_state",
+ lambda _gc, marker: not (
+ rollback_failure == "final-persist" and marker is None
+ ),
+ )
+
+ result = m._planned_mysql_global_writer_switch(_mysql_switchover_params())
+
+ assert result[0] == 400
+ assert writer_member["IsWriter"] is True
+ assert target_member["IsWriter"] is False
+ assert target in configured
+ assert writable == (
+ []
+ if rollback_failure in (
+ "source-reset", "target-restore", "target-health", "target-fence",
+ )
+ else [writer]
+ )
+ assert target_safety == (
+ [target]
+ if rollback_failure == "target-fence"
+ else [target, writer, target]
+ if rollback_failure == "final-persist"
+ else []
+ )
+ owner = m._active_mysql_global_writer_switch(global_cluster)
+ assert owner is not None and owner["state"] == "repair_required"
+ marker = global_cluster[m._MYSQL_GLOBAL_SWITCHOVER_STATE]
+ assert marker["state"] == "repair_required"
+ assert marker["source_arn"] == writer["DBClusterArn"]
+ assert marker["target_arn"] == target["DBClusterArn"]
+ m._mysql_global_writer_switch_owners.clear()
+ assert m._active_mysql_global_writer_switch(global_cluster) is marker
+ assert m._mysql_global_writer_switch_mutation_error(writer)[0] == 400
+ assert global_cluster["Status"] == "available"
+ assert b"<Status>switching-over</Status>" in m._describe_global_clusters({
+ "GlobalClusterIdentifier": "global-repl",
+ })[2]
+
+
+def test_rds_switchover_deadline_starts_before_preflight(monkeypatch):
+ from ministack.services import rds as m
+
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ now = [100.0]
+ deadlines = []
+ monkeypatch.setattr(m.time, "monotonic", lambda: now[0])
+
+ def ensure_control(*_args, **kwargs):
+ deadlines.append(kwargs["deadline"])
+ now[0] += 2
+ return len(deadlines) == 1
+
+ monkeypatch.setattr(m, "_ensure_mysql_control_user", ensure_control)
+
+ result = m._planned_mysql_global_writer_switch(_mysql_switchover_params())
+
+ assert result[0] == 400
+ assert deadlines == [130.0, 130.0]
+
+
+def test_rds_switchover_initial_lock_respects_deadline(monkeypatch):
+ from ministack.services import rds as m
+
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ acquired = threading.Event()
+ release = threading.Event()
+
+ def hold_lifecycle_lock():
+ with m._shared_container_lock:
+ acquired.set()
+ release.wait(timeout=1)
+
+ holder = threading.Thread(target=hold_lifecycle_lock)
+ holder.start()
+ assert acquired.wait(timeout=1)
+ monkeypatch.setattr(m, "_MYSQL_GLOBAL_SWITCHOVER_TIMEOUT", 0.02)
+ monkeypatch.setattr(
+ m,
+ "_resolve_global_writer_switch",
+ lambda *_a, **_k: pytest.fail("timed-out request resolved topology"),
+ )
+
+ try:
+ started = time.monotonic()
+ result = m._planned_mysql_global_writer_switch(
+ _mysql_switchover_params(),
+ )
+ elapsed = time.monotonic() - started
+ finally:
+ release.set()
+ holder.join(timeout=1)
+
+ assert result[0] == 400
+ assert elapsed < 0.5
+ assert not holder.is_alive()
+
+
+def test_rds_switchover_lock_deadline_never_claims_owner(monkeypatch):
+ from ministack.services import rds as m
+
+ writer, target, writer_member, target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+ acquired = threading.Event()
+ release = threading.Event()
+ holder = None
+
+ def hold_lifecycle_lock():
+ with m._shared_container_lock:
+ acquired.set()
+ release.wait(timeout=1)
+
+ def ensure_replication_user(*_args, **_kwargs):
+ nonlocal holder
+ holder = threading.Thread(target=hold_lifecycle_lock)
+ holder.start()
+ assert acquired.wait(timeout=1)
+ return True
+
+ monkeypatch.setattr(m, "_MYSQL_GLOBAL_SWITCHOVER_TIMEOUT", 0.02)
+ monkeypatch.setattr(m, "_ensure_mysql_control_user", lambda *_a, **_k: True)
+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", ensure_replication_user)
+ monkeypatch.setattr(m, "_mysql_writer_fence_active", lambda *_a, **_k: True)
+
+ try:
+ result = m._planned_mysql_global_writer_switch(
+ _mysql_switchover_params(),
+ )
+ finally:
+ release.set()
+ if holder:
+ holder.join(timeout=1)
+
+ assert result[0] == 400
+ assert m._active_mysql_global_writer_switch(global_cluster) is None
+ assert writer_member["IsWriter"] is True
+ assert target_member["IsWriter"] is False
+ assert writer.get("_mysql_replication_source_arn") is None
+ assert target["_mysql_replication_source_arn"] == writer["DBClusterArn"]
+
+
+def test_rds_mysql_gtid_wait_recomputes_shared_deadline(monkeypatch):
+ from ministack.services import rds as m
+
+ now = [100.0]
+ observed = {}
+ monkeypatch.setattr(m.time, "monotonic", lambda: now[0])
+
+ class Cursor:
+ def __init__(self, connection):
+ self.connection = connection
+
+ def execute(self, statement, params):
+ observed["sql"] = (statement, params)
+
+ def fetchone(self):
+ return (0,)
+
+ def close(self):
+ pass
+
+ class Connection:
+ def cursor(self):
+ now[0] += 4
+ return Cursor(self)
+
+ def close(self):
+ pass
+
+ def connect(_cluster, *, timeout):
+ observed["connect_timeout"] = timeout
+ return Connection()
+
+ monkeypatch.setattr(m, "_mysql_replication_connection", connect)
+
+ assert m._wait_for_mysql_gtid(
+ {"DBClusterIdentifier": "secondary"}, "source:1-9", 30,
+ deadline=130.0,
+ ) is True
+ assert observed["connect_timeout"] == 30.0
+ assert observed["sql"][1] == ("source:1-9", 26.0)
+
+
+def test_rds_mysql_replication_health_waits_for_expected_running_source(monkeypatch):
+ from ministack.services import rds as m
+
+ rows = iter([
+ ("Connecting", "Yes", "172.20.0.20", 3306),
+ ("Yes", "Yes", "172.20.0.20", 3306),
+ ])
+ statements = []
+ sleeps = []
+
+ class Cursor:
+ description = [
+ ("Replica_IO_Running",),
+ ("Replica_SQL_Running",),
+ ("Source_Host",),
+ ("Source_Port",),
+ ]
+
+ def __init__(self, connection):
+ self.connection = connection
+
+ def execute(self, statement):
+ statements.append(statement)
+
+ def fetchone(self):
+ return next(rows)
+
+ def close(self):
+ pass
+
+ class Connection:
+ def cursor(self):
+ return Cursor(self)
+
+ def close(self):
+ pass
+
+ monkeypatch.setattr(
+ m, "_mysql_replication_connection", lambda *_a, **_k: Connection(),
+ )
+ monkeypatch.setattr(m.time, "sleep", sleeps.append)
+ monkeypatch.setattr(m.time, "monotonic", lambda: 100.0)
+
+ assert m._wait_for_mysql_replication_healthy(
+ {"DBClusterIdentifier": "primary"},
+ {"_shared_internal_address": "172.20.0.20", "_shared_internal_port": 3306},
+ deadline=130.0,
+ ) is True
+ assert statements == ["SHOW REPLICA STATUS"] * 2
+ assert sleeps == [m._MYSQL_REPLICATION_RETRY_INTERVAL]
+
+
+def test_rds_mysql_replication_health_rejects_wrong_source_at_deadline(monkeypatch):
+ from ministack.services import rds as m
+
+ now = iter([100.0, 100.0, 130.0])
+
+ class Cursor:
+ description = [
+ ("Replica_IO_Running",),
+ ("Replica_SQL_Running",),
+ ("Source_Host",),
+ ("Source_Port",),
+ ]
+
+ def __init__(self, connection):
+ self.connection = connection
+
+ def execute(self, _statement):
+ pass
+
+ def fetchone(self):
+ return ("Yes", "Yes", "unexpected", 3306)
+
+ def close(self):
+ pass
+
+ class Connection:
+ def cursor(self):
+ return Cursor(self)
+
+ def close(self):
+ pass
+
+ monkeypatch.setattr(
+ m, "_mysql_replication_connection", lambda *_a, **_k: Connection(),
+ )
+ monkeypatch.setattr(m.time, "monotonic", lambda: next(now))
+ monkeypatch.setattr(
+ m.time, "sleep", lambda *_a: pytest.fail("expired health check slept"),
+ )
+
+ assert m._wait_for_mysql_replication_healthy(
+ {"DBClusterIdentifier": "primary"},
+ {"_shared_internal_address": "172.20.0.20", "_shared_internal_port": 3306},
+ deadline=130.0,
+ ) is False
+
+
[email protected]("mutation", ["password", "topology", "failover"])
+def test_rds_active_switchover_rejects_mutation(monkeypatch, mutation):
+ from ministack.services import rds as m
+
+ writer, target, writer_member, target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+ with m._shared_container_lock:
+ owner = m._claim_mysql_global_writer_switch(global_cluster, writer, target)
+ if mutation == "password":
+ result = m._modify_db_cluster({
+ "DBClusterIdentifier": "primary",
+ "MasterUserPassword": "changed-password",
+ })
+ elif mutation == "topology":
+ result = m._remove_from_global_cluster({
+ "GlobalClusterIdentifier": "global-repl",
+ "DbClusterIdentifier": "secondary",
+ })
+ else:
+ result = m._failover_global_cluster({
+ "GlobalClusterIdentifier": "global-repl",
+ "TargetDbClusterIdentifier": "secondary",
+ "AllowDataLoss": "true",
+ })
+
+ assert result[0] == 400
+ assert m._active_mysql_global_writer_switch(global_cluster) is owner
+ assert writer_member["IsWriter"] is True
+ assert target_member["IsWriter"] is False
+ assert writer.get("_MasterUserPassword") != "changed-password"
+ assert target["GlobalClusterIdentifier"] == "global-repl"
+ assert target["_mysql_replication_source_arn"] == writer["DBClusterArn"]
+
+
+ ("handler_name", "initial_status"),
+ [
+ ("_start_db_instance", "stopped"),
+ ("_stop_db_instance", "available"),
+ ("_reboot_db_instance", "stopped"),
+ ],
+)
+def test_rds_active_switchover_rejects_member_status_mutation(
+ monkeypatch,
+ handler_name,
+ initial_status,
+):
+ from ministack.services import rds as m
+
+ writer, target, _writer_member, _target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+ instance = {
+ "DBInstanceIdentifier": "instance-1",
+ "DBClusterIdentifier": "primary",
+ "DBInstanceStatus": initial_status,
+ }
+ monkeypatch.setattr(m, "_resolve_instance", lambda _identifier: instance)
+ with m._shared_container_lock:
+ owner = m._claim_mysql_global_writer_switch(global_cluster, writer, target)
+
+ result = getattr(m, handler_name)({"DBInstanceIdentifier": "instance-1"})
+
+ assert result[0] == 400
+ assert instance["DBInstanceStatus"] == initial_status
+ assert m._active_mysql_global_writer_switch(global_cluster) is owner
+
+
+ ("handler_name", "impl_name"),
+ [
+ ("_create_db_instance", "_create_db_instance_impl"),
+ ("_delete_db_instance", "_delete_db_instance_impl"),
+ ],
+)
+def test_rds_instance_mutation_prevents_switchover_claim(
+ monkeypatch,
+ handler_name,
+ impl_name,
+):
+ from ministack.services import rds as m
+
+ writer, target, _writer_member, _target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+ entered = threading.Event()
+ release = threading.Event()
+ claimed = threading.Event()
+ order = []
+
+ def mutation_impl(_params):
entered.set()
assert release.wait(timeout=1)
order.append("mutation")
@@ -10998,6 +12177,139 @@ def test_aurora_mysql_control_user_can_inventory_writer_transactions():
_delete_global_cluster(east, global_id)
+ not os.environ.get("DOCKER_NETWORK"),
+ reason="DOCKER_NETWORK not set -- live Aurora global switchover",
+)
+def test_aurora_mysql_global_switchover_relinks_data_plane():
+ import pymysql
+
+ east = _regional_rds("us-east-1")
+ west = _regional_rds("us-west-2")
+ suffix = uuid.uuid4().hex[:10]
+ global_id = f"global-switch-live-{suffix}"
+ primary_id = f"global-switch-live-primary-{suffix}"
+ primary_instance_id = f"{primary_id}-writer"
+ secondary_id = f"global-switch-live-secondary-{suffix}"
+ secondary_instance_id = f"{secondary_id}-reader"
+ primary_arn = None
+ secondary_arn = None
+
+ try:
+ primary = east.create_db_cluster(
+ DBClusterIdentifier=primary_id,
+ Engine="aurora-mysql",
+ EngineVersion="8.0.mysql_aurora.3.10.3",
+ MasterUsername="admin",
+ MasterUserPassword=PASSWORD,
+ DatabaseName=DATABASE,
+ )["DBCluster"]
+ primary_arn = primary["DBClusterArn"]
+ east.create_db_instance(
+ DBInstanceIdentifier=primary_instance_id,
+ DBClusterIdentifier=primary_id,
+ DBInstanceClass="db.r6g.large",
+ Engine="aurora-mysql",
+ )
+ primary_instance = _wait_for_instance(east, primary_instance_id)
+ east.create_global_cluster(
+ GlobalClusterIdentifier=global_id,
+ SourceDBClusterIdentifier=primary_arn,
+ )
+ secondary = west.create_db_cluster(
+ DBClusterIdentifier=secondary_id,
+ Engine="aurora-mysql",
+ EngineVersion="8.0.mysql_aurora.3.10.3",
+ GlobalClusterIdentifier=global_id,
+ MasterUsername="admin",
+ MasterUserPassword=PASSWORD,
+ DatabaseName=DATABASE,
+ )["DBCluster"]
+ secondary_arn = secondary["DBClusterArn"]
+ west.create_db_instance(
+ DBInstanceIdentifier=secondary_instance_id,
+ DBClusterIdentifier=secondary_id,
+ DBInstanceClass="db.r6g.large",
+ Engine="aurora-mysql",
+ )
+ secondary_instance = _wait_for_instance(west, secondary_instance_id)
+ _wait_for_replica_status(secondary_instance["Endpoint"])
+
+ with _aurora_connect(primary_instance["Endpoint"]) as conn:
+ with conn.cursor() as cursor:
+ cursor.execute(
+ "CREATE TABLE switchover_rows "
+ "(id INT PRIMARY KEY, value VARCHAR(32))"
+ )
+ cursor.execute(
+ "INSERT INTO switchover_rows VALUES (1, 'old-writer')"
+ )
+ _wait_for_gtid(
+ primary_instance["Endpoint"], secondary_instance["Endpoint"],
+ )
+
+ response = east.switchover_global_cluster(
+ GlobalClusterIdentifier=global_id,
+ TargetDbClusterIdentifier=secondary_arn,
+ )["GlobalCluster"]
+ assert response["Status"] == "switching-over"
+ assert response["FailoverState"] == {
+ "Status": "pending",
+ "FromDbClusterArn": primary_arn,
+ "ToDbClusterArn": secondary_arn,
+ "IsDataLossAllowed": False,
+ }
+ members = {
+ member["DBClusterArn"]: member
+ for member in response["GlobalClusterMembers"]
+ }
+ assert members[primary_arn]["IsWriter"] is False
+ assert members[secondary_arn]["IsWriter"] is True
+ final = west.describe_global_clusters(
+ GlobalClusterIdentifier=global_id,
+ )["GlobalClusters"][0]
+ final_members = {
+ member["DBClusterArn"]: member
+ for member in final["GlobalClusterMembers"]
+ }
+ assert final["Status"] == "available"
+ assert "FailoverState" not in final
+ assert final_members[primary_arn]["IsWriter"] is False
+ assert final_members[secondary_arn]["IsWriter"] is True
+ assert final_members[secondary_arn]["Readers"] == [primary_arn]
+
+ with _aurora_connect(secondary_instance["Endpoint"]) as conn:
+ with conn.cursor() as cursor:
+ cursor.execute("SELECT id, value FROM switchover_rows")
+ assert cursor.fetchall() == ((1, "old-writer"),)
+ cursor.execute(
+ "INSERT INTO switchover_rows VALUES (2, 'new-writer')"
+ )
+ _wait_for_gtid(
+ secondary_instance["Endpoint"], primary_instance["Endpoint"],
+ )
+ with _aurora_connect(primary_instance["Endpoint"]) as conn:
+ with conn.cursor() as cursor:
+ cursor.execute(
+ "SELECT id, value FROM switchover_rows ORDER BY id"
+ )
+ assert cursor.fetchall() == (
+ (1, "old-writer"),
+ (2, "new-writer"),
+ )
+ with pytest.raises(pymysql.err.OperationalError) as exc:
+ cursor.execute(
+ "INSERT INTO switchover_rows VALUES (3, 'old-write')"
+ )
+ assert exc.value.args[0] == 1290
+ finally:
+ _cleanup_two_member_global(east, global_id, primary_arn, secondary_arn)
+ _delete_instance(west, secondary_instance_id)
+ _delete_instance(east, primary_instance_id)
+ _delete_cluster(west, secondary_id)
+ _delete_cluster(east, primary_id)
+
+
@pytest.mark.skipif(
not os.environ.get("DOCKER_NETWORK"),
reason="DOCKER_NETWORK not set -- live Aurora global replication",