Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
feat(rds): add planned Aurora MySQL switchover (#1621)
Ian Oberst57a98b686d08
diff --git a/ministack/services/rds.py b/ministack/services/rds.pyindex 51d33a33..6c0eff8e 100644--- a/ministack/services/rds.py+++ b/ministack/services/rds.py@@ -54,7 +54,7 @@ import timefrom urllib.parse import parse_qsfrom xml.sax.saxutils import escape as _esc-from ministack.core import container_reaper+from ministack.core import container_reaper, persistencefrom ministack.core.arn import ArnParseError, parse_arnfrom ministack.core.concurrency import resource_lock, run_offloop, spawn_backgroundfrom ministack.core.persistence import load_state@@ -130,6 +130,8 @@ _MYSQL_CONTROL_USER = "rdsadmin"_MYSQL_CONTROL_PASSWORD = "ministack-rds-control"_MYSQL_REPLICATION_RETRY_ATTEMPTS = 60_MYSQL_REPLICATION_RETRY_INTERVAL = 1+_MYSQL_GLOBAL_SWITCHOVER_TIMEOUT = 30+_MYSQL_GLOBAL_SWITCHOVER_STATE = "_mysql_global_writer_switch_state"_MYSQL_TRANSACTION_STATES = frozenset({"RUNNING", "LOCK WAIT", "ROLLING BACK", "COMMITTING"})_MYSQL_MODIFIED_TRANSACTION_QUERY = ("SELECT i.TRX_ID, i.TRX_STATE, i.TRX_STARTED, i.TRX_MYSQL_THREAD_ID, "@@ -391,6 +393,13 @@ def restore_state(data):_db_proxies.update(data.get("db_proxies", {}))_db_proxy_endpoints.update(data.get("db_proxy_endpoints", {}))_global_clusters.update(data.get("global_clusters", {}))+ for global_cluster in _global_clusters._data.values():+ marker = global_cluster.get(_MYSQL_GLOBAL_SWITCHOVER_STATE)+ if isinstance(marker, dict):+ # A live request owner cannot survive a process restart. Any+ # persisted in-progress operation therefore becomes an explicit+ # repair state rather than being resumed from an unknown phase.+ marker["state"] = "repair_required"# Persistence contains every account, while AccountScopedDict.values()# intentionally exposes only the active request account. Reconcile the# raw persisted records so a warm boot cannot leave secondary metadata@@ -808,6 +817,14 @@ def restore_state(data):cluster_id,engine=cluster.get("Engine", "aurora-mysql"),)+ repair_required = (+ _mysql_global_writer_switch_repair_required(cluster)+ )+ repair_fence_verified = bool(+ authenticated_ready+ and repair_required+ and _mysql_writer_fence_active(cluster) is True+ )with _shared_container_lock:current_cluster = _clusters.get(cluster_id)if (@@ -820,6 +837,26 @@ def restore_state(data):or cluster.get("_shared_container_id") != container_id):return+ if repair_required:+ if authenticated_ready and not repair_fence_verified:+ logger.error(+ "RDS: stopping restored MySQL compute for %s "+ "because its repair fence could not be verified",+ cluster_id,+ )+ if not _contain_cluster_compute_preserving_volume(+ cluster_id, cluster,+ ):+ logger.critical(+ "RDS: unfenced repair compute for %s could "+ "not be contained",+ cluster_id,+ )+ # Keep a verified-fenced server reachable only as+ # repair compute. Do not advertise it as ready or run+ # normal replication reconciliation against an+ # uncertain channel topology.+ authenticated_ready = Falsepending_rotation = cluster.get("_pending_master_password_rotation",)@@ -882,7 +919,12 @@ def restore_state(data):# that transition.breakelse:- cluster["_shared_container_ready"] = status == "available"+ if _mysql_global_writer_switch_repair_required(cluster):+ cluster["_shared_container_ready"] = False+ _set_cluster_members_status(cluster, "failed")+ _refresh_cluster_status(cluster_id)+ else:+ cluster["_shared_container_ready"] = status == "available"# Superseded member volumes are recovery copies until the adopted# writer volume has passed an authenticated database readiness@@ -1100,6 +1142,17 @@ def _start_cluster_shared_container(cluster_id, cluster, remove_stale=False):"--log-replica-updates",f"--binlog-expire-logs-seconds={_MYSQL_BINLOG_RETENTION_SECONDS}",]+ if (+ _is_mysql_engine(engine)+ and _mysql_global_writer_switch_repair_required(cluster)+ ):+ # SET GLOBAL fences do not reliably survive a container restart. A+ # persisted repair marker therefore has to fence the server before it+ # accepts its first connection, not after readiness publishes it.+ container_kwargs.setdefault("command", []).extend([+ "--read-only=ON",+ "--super-read-only=ON",+ ])if ms_network:container_kwargs["network"] = ms_network# Aurora storage belongs to the cluster, not to any member instance. Use a@@ -1860,6 +1913,109 @@ def _stop_cluster_shared_container(cluster_id, cluster):return True+def _force_remove_cluster_compute_preserving_volume(cluster_id, cluster):+ """Force-remove owned cluster compute without deleting durable storage."""+ parsed = _parse_rds_arn(cluster.get("DBClusterArn", ""))+ if not parsed or parsed[1] != "cluster":+ logger.error(+ "RDS: cannot prove repair compute scope for cluster %s",+ cluster_id,+ )+ return False+ scope, _resource_type, parsed_cluster_id = parsed+ if parsed_cluster_id != cluster_id:+ logger.error(+ "RDS: repair compute identifier %s does not match ARN for %s",+ cluster_id,+ parsed_cluster_id,+ )+ return False+ docker_client = _get_docker()+ if not docker_client:+ return False+ identifiers = []+ if cluster.get("_shared_container_id"):+ identifiers.append(cluster["_shared_container_id"])+ container_name = _rds_cluster_docker_name(+ cluster_id, scope.account_id, scope.region,+ )+ if container_name not in identifiers:+ identifiers.append(container_name)++ def _not_found(error):+ response = getattr(error, "response", None)+ return (+ getattr(error, "status_code", None) == 404+ or getattr(response, "status_code", None) == 404+ )++ for identifier in identifiers:+ try:+ container = docker_client.containers.get(identifier)+ except Exception as e:+ if _not_found(e):+ continue+ logger.error(+ "RDS: failed to inspect repair container %s for cluster %s: %s",+ identifier,+ cluster_id,+ e,+ )+ return False+ if not _rds_container_is_owned_by(+ container,+ expected_cluster_ids={cluster_id},+ account_id=scope.account_id,+ region=scope.region,+ ):+ logger.error(+ "RDS: refusing to force-remove unowned repair container %s "+ "for cluster %s",+ identifier,+ cluster_id,+ )+ return False+ try:+ container.remove(force=True, v=False)+ except Exception as e:+ logger.error(+ "RDS: failed to force-remove repair container %s for cluster "+ "%s: %s",+ identifier,+ cluster_id,+ e,+ )+ return False+ for identifier in identifiers:+ try:+ docker_client.containers.get(identifier)+ except Exception as e:+ if _not_found(e):+ continue+ logger.error(+ "RDS: failed to verify repair container %s removal for cluster "+ "%s: %s",+ identifier,+ cluster_id,+ e,+ )+ return False+ logger.error(+ "RDS: repair container %s for cluster %s remains after force-remove",+ identifier,+ cluster_id,+ )+ return False+ cluster["_shared_container_id"] = None+ cluster["_shared_internal_address"] = None+ cluster["_shared_internal_port"] = None+ return True+++def _contain_cluster_compute_preserving_volume(cluster_id, cluster):+ return _force_remove_cluster_compute_preserving_volume(cluster_id, cluster)++def _remove_cluster_shared_resources(cluster_id, cluster, timeout=5, account_id=None, region=None,):@@ -2994,6 +3150,57 @@ def _ensure_mysql_replication_user(cluster, *, timeout=None, deadline=None):return succeeded+def _set_mysql_cluster_fenced(cluster, *, timeout=None, deadline=None):+ """Install and verify a writer fence using bounded control connections."""+ conn = None+ succeeded = False+ try:+ if deadline is None and timeout is not None:+ deadline = time.monotonic() + max(float(timeout), 0.0)+ conn = _connect_mysql(+ _mysql_replication_connection,+ cluster,+ _mysql_deadline_timeout(timeout=timeout, deadline=deadline),+ )+ if conn is None:+ return False+ succeeded = _set_mysql_writer_fence(conn, deadline=deadline)+ if succeeded:+ succeeded = _mysql_writer_fence_active(+ cluster, deadline=deadline,+ ) is True+ except Exception as e:+ logger.warning(+ "RDS: failed to fence MySQL cluster %s: %s",+ cluster.get("DBClusterIdentifier", "unknown"),+ e,+ )+ finally:+ if not _close_mysql_resources("cluster fence", connection=conn):+ succeeded = False+ return succeeded+++def _fence_or_contain_mysql_cluster(cluster, *, timeout=3):+ """Return ``fenced`` or ``contained`` only after proving one is true."""+ if _set_mysql_cluster_fenced(cluster, timeout=timeout):+ return "fenced"+ cluster_id = cluster.get("DBClusterIdentifier", "")+ with _shared_container_lock:+ contained = _contain_cluster_compute_preserving_volume(+ cluster_id, cluster,+ )+ _set_cluster_members_status(cluster, "failed")+ cluster["Status"] = "creating"+ if contained:+ return "contained"+ logger.critical(+ "RDS: unfenced MySQL compute for %s could not be contained",+ cluster_id or "unknown",+ )+ return None++def _configure_mysql_replica_from_source(cluster_id, replica, source, *, timeout=None, deadline=None,):@@ -3921,10 +4128,8 @@ def _resolve_instance(db_id):def _set_cluster_members_status(cluster, status):- for member in cluster.get("DBClusterMembers", []):- instance = _instances.get(member.get("DBInstanceIdentifier"))- if instance is not None:- instance["DBInstanceStatus"] = status+ for instance in _cluster_member_instances(cluster):+ instance["DBInstanceStatus"] = statusdef _attach_instance_to_shared_cluster(instance, cluster):@@ -4979,7 +5184,7 @@ def _modify_db_instance(p):return _single_instance_response("ModifyDBInstanceResponse", "ModifyDBInstanceResult", instance)-def _start_db_instance(p):+def _set_db_instance_status(p, status, response_name, result_name):db_id = _p(p, "DBInstanceIdentifier")instance = _resolve_instance(db_id)if not instance:@@ -4987,32 +5192,36 @@ def _start_db_instance(p):if invalid_arn:return invalid_arnreturn _error("DBInstanceNotFound", f"DBInstance {db_id} not found.", 404)- instance["DBInstanceStatus"] = "available"- return _single_instance_response("StartDBInstanceResponse", "StartDBInstanceResult", instance)+ cluster_id = (+ instance.get("_shared_cluster_id") or instance.get("DBClusterIdentifier")+ )+ cluster = _resolve_cluster_in_request_region(cluster_id) if cluster_id else None+ if cluster and (switch_error := _begin_mysql_global_instance_mutation(cluster)):+ return switch_error+ try:+ instance["DBInstanceStatus"] = status+ return _single_instance_response(response_name, result_name, instance)+ finally:+ if cluster:+ _end_mysql_global_instance_mutation(cluster)+++def _start_db_instance(p):+ return _set_db_instance_status(+ p, "available", "StartDBInstanceResponse", "StartDBInstanceResult",+ )def _stop_db_instance(p):- db_id = _p(p, "DBInstanceIdentifier")- instance = _resolve_instance(db_id)- if not instance:- invalid_arn = _invalid_db_instance_identifier_error(db_id)- if invalid_arn:- return invalid_arn- return _error("DBInstanceNotFound", f"DBInstance {db_id} not found.", 404)- instance["DBInstanceStatus"] = "stopped"- return _single_instance_response("StopDBInstanceResponse", "StopDBInstanceResult", instance)+ return _set_db_instance_status(+ p, "stopped", "StopDBInstanceResponse", "StopDBInstanceResult",+ )def _reboot_db_instance(p):- db_id = _p(p, "DBInstanceIdentifier")- instance = _resolve_instance(db_id)- if not instance:- invalid_arn = _invalid_db_instance_identifier_error(db_id)- if invalid_arn:- return invalid_arn- return _error("DBInstanceNotFound", f"DBInstance {db_id} not found.", 404)- instance["DBInstanceStatus"] = "available"- return _single_instance_response("RebootDBInstanceResponse", "RebootDBInstanceResult", instance)+ return _set_db_instance_status(+ p, "available", "RebootDBInstanceResponse", "RebootDBInstanceResult",+ )# ---------------------------------------------------------------------------@@ -7223,8 +7432,8 @@ def _invalid_global_cluster_identifier_error(global_id):# Global Clusters## Emulation scope: Aurora Global Database membership plus native MySQL 8 GTID-# replication between provisioned members. Failover and switchover remain-# metadata-only; their data-plane promotion behavior is a separate phase.+# replication between provisioned members. Planned Aurora MySQL 8 switchovers+# move the data plane synchronously; lossy failover remains metadata-only.# ---------------------------------------------------------------------------def _create_global_cluster(p):@@ -7492,11 +7701,113 @@ def _find_global_cluster_target_member(gc, target_cluster_id):return cluster, member+def _resolve_global_writer_switch(p, *, exact_two=False):+ gc_id = _p(p, "GlobalClusterIdentifier")+ target_id = _p(p, "TargetDbClusterIdentifier")+ invalid_id = _invalid_global_cluster_identifier_error(gc_id)+ if invalid_id:+ return None, invalid_id+ gc = _resolve_global_cluster(gc_id)+ if not gc:+ return None, _error(+ "GlobalClusterNotFoundFault", f"Global cluster {gc_id} not found.", 404,+ )+ if not target_id:+ return None, _error("MissingParameter", "TargetDbClusterIdentifier is required", 400)+ target, target_member = _find_global_cluster_target_member(gc, target_id)+ if not target_member:+ if not _resolve_cluster(target_id):+ return None, _error(+ "DBClusterNotFoundFault", f"DBCluster {target_id} not found.", 404,+ )+ return None, _error(+ "InvalidGlobalClusterStateFault",+ f"DBCluster {target_id} is not a secondary member of global "+ f"cluster {gc_id}.",+ 400,+ )+ members = gc.get("GlobalClusterMembers", [])+ writer = next((member for member in members if member.get("IsWriter")), None)+ wrong_size = len(members) != 2 if exact_two else len(members) < 2+ if not writer or target_member.get("IsWriter") or wrong_size:+ requirement = "one writer and one secondary" if exact_two else "a secondary"+ return None, _error(+ "InvalidGlobalClusterStateFault",+ f"Global cluster {gc_id} must have {requirement} target to promote.",+ 400,+ )+ return (gc, target, target_member, writer), None+++def _persisted_mysql_global_writer_switch_state(gc):+ marker = gc.get(_MYSQL_GLOBAL_SWITCHOVER_STATE) if gc else None+ if (+ isinstance(marker, dict)+ and marker.get("state") in ("in_progress", "repair_required")+ ):+ return marker+ return None+++def _persisted_mysql_global_writer_switch_cluster(data, gc):+ if not isinstance(data, dict):+ return None+ global_clusters = data.get("global_clusters")+ if isinstance(global_clusters, (AccountScopedDict, AccountRegionScopedDict)):+ candidates = global_clusters.values()+ elif isinstance(global_clusters, dict):+ candidates = global_clusters.values()+ else:+ return None+ identifier = gc.get("GlobalClusterIdentifier")+ arn = gc.get("GlobalClusterArn")+ for candidate in candidates:+ if not isinstance(candidate, dict):+ continue+ if candidate.get("GlobalClusterIdentifier") != identifier:+ continue+ if arn and candidate.get("GlobalClusterArn") != arn:+ continue+ return candidate+ return None+++def _persisted_mysql_global_writer_switch_marker(data, gc):+ persisted_gc = _persisted_mysql_global_writer_switch_cluster(data, gc)+ if persisted_gc is None:+ return None+ return persisted_gc.get(_MYSQL_GLOBAL_SWITCHOVER_STATE)+++def _durably_persist_mysql_global_writer_switch_state(gc, expected_marker):+ """Write and verify the recovery state around data-plane changes."""+ if not persistence.PERSIST_STATE:+ return True+ persistence.save_state("rds", get_state())+ persisted = persistence.load_state("rds")+ persisted_gc = _persisted_mysql_global_writer_switch_cluster(persisted, gc)+ if (+ persisted_gc is not None+ and persisted_gc.get(_MYSQL_GLOBAL_SWITCHOVER_STATE) == expected_marker+ ):+ return True+ logger.error(+ "RDS: global writer switchover recovery state could not be durably "+ "persisted"+ )+ return False+++def _mysql_global_writer_switch_repair_required(cluster):+ gc, _member = _global_cluster_member_for_cluster(cluster)+ return _persisted_mysql_global_writer_switch_state(gc) is not None++def _active_mysql_global_writer_switch(gc):owner = _mysql_global_writer_switch_owners.get(id(gc)) if gc else Noneif owner and owner.get("global_cluster") is gc:return owner- return None+ return _persisted_mysql_global_writer_switch_state(gc)def _mysql_global_writer_switch_mutation_error(cluster):@@ -7589,6 +7900,18 @@ def _mysql_global_writer_switch_ready(gc, source, target):)+def _mysql_global_writer_switch_owner(gc, source, target, operation_id):+ return {+ "global_cluster": gc,+ "source": source,+ "target": target,+ "topology": _mysql_global_writer_switch_topology(gc),+ "source_identity": _mysql_global_writer_switch_identity(source),+ "target_identity": _mysql_global_writer_switch_identity(target),+ "operation_id": operation_id,+ }++def _claim_mysql_global_writer_switch(gc, source, target):with _mysql_global_writer_switch_lock:if _active_mysql_global_writer_switch(gc) or any(@@ -7596,18 +7919,116 @@ def _claim_mysql_global_writer_switch(gc, source, target):for cluster in (source, target)):return None- owner = {- "global_cluster": gc,- "source": source,- "target": target,- "topology": _mysql_global_writer_switch_topology(gc),- "source_identity": _mysql_global_writer_switch_identity(source),- "target_identity": _mysql_global_writer_switch_identity(target),+ operation_id = new_uuid()+ owner = _mysql_global_writer_switch_owner(+ gc, source, target, operation_id,+ )+ marker = {+ "state": "in_progress",+ "source_arn": source.get("DBClusterArn"),+ "target_arn": target.get("DBClusterArn"),+ "operation_id": operation_id,}+ gc[_MYSQL_GLOBAL_SWITCHOVER_STATE] = marker+ if not _durably_persist_mysql_global_writer_switch_state(gc, marker):+ gc.pop(_MYSQL_GLOBAL_SWITCHOVER_STATE, None)+ return None_mysql_global_writer_switch_owners[id(gc)] = ownerreturn owner+def _mysql_global_writer_switch_repair_matches(marker, source, target):+ return bool(+ isinstance(marker, dict)+ and marker.get("state") == "repair_required"+ and marker.get("operation_id")+ and marker.get("source_arn") == source.get("DBClusterArn")+ and marker.get("target_arn") == target.get("DBClusterArn")+ )+++def _mysql_global_writer_switch_repair_ready(gc, source, target, marker):+ return bool(+ _mysql_global_writer_switch_repair_matches(marker, source, target)+ and gc.get(_MYSQL_GLOBAL_SWITCHOVER_STATE) is marker+ and source.get("EngineVersion") == target.get("EngineVersion")+ and _mysql_gtid_history_ready(source)+ and _mysql_gtid_history_ready(target)+ and all(+ cluster.get("_shared_storage_initialized")+ and cluster.get("DBClusterMembers")+ and cluster.get("_shared_container_id")+ and cluster.get("_shared_internal_address")+ and cluster.get("_shared_internal_port")+ and isinstance(cluster.get("_shared_container_epoch"), int)+ and cluster.get("_shared_container_epoch") > 0+ and not cluster.get("_pending_master_password_rotation")+ and not _active_mysql_global_instance_mutation(cluster)+ for cluster in (source, target)+ )+ )+++def _resume_mysql_global_writer_switch(gc, source, target, marker):+ """Reclaim a restored repair marker for a retry of the same switchover."""+ with _mysql_global_writer_switch_lock:+ if id(gc) in _mysql_global_writer_switch_owners:+ return None+ if not _mysql_global_writer_switch_repair_ready(+ gc, source, target, marker,+ ):+ return None+ for cluster in (source, target):+ cluster["_shared_container_ready"] = True+ cluster["Status"] = "available"+ _set_cluster_members_status(cluster, "available")+ gc["Status"] = "available"+ marker["state"] = "in_progress"+ owner = _mysql_global_writer_switch_owner(+ gc, source, target, marker["operation_id"],+ )+ _mysql_global_writer_switch_owners[id(gc)] = owner+ return owner+++def _resolve_committed_mysql_global_writer_switch_repair(p):+ gc = _resolve_global_cluster(_p(p, "GlobalClusterIdentifier"))+ marker = _persisted_mysql_global_writer_switch_state(gc)+ if not gc or not marker or marker.get("state") != "repair_required":+ return None+ members = gc.get("GlobalClusterMembers", [])+ source_member = next(+ (+ member for member in members+ if member.get("DBClusterArn") == marker.get("source_arn")+ ),+ None,+ )+ target_member = next(+ (+ member for member in members+ if member.get("DBClusterArn") == marker.get("target_arn")+ ),+ None,+ )+ source = _resolve_global_member_cluster(source_member)+ target = _resolve_global_member_cluster(target_member)+ requested_target = _p(p, "TargetDbClusterIdentifier")+ if (+ not source+ or not target+ or not source_member+ or not target_member+ or source_member.get("IsWriter")+ or not target_member.get("IsWriter")+ or requested_target not in (+ target.get("DBClusterIdentifier"), target.get("DBClusterArn"),+ )+ ):+ return None+ return gc, source, target, source_member, target_member, marker++def _mysql_global_writer_switch_still_owned(owner, expected_topology):gc = owner["global_cluster"]source = owner["source"]@@ -7632,44 +8053,39 @@ def _finish_mysql_global_writer_switch(owner, *, repair_required=False):returnif repair_required:owner["state"] = "repair_required"+ marker = gc.get(_MYSQL_GLOBAL_SWITCHOVER_STATE)+ if not isinstance(marker, dict):+ marker = {}+ gc[_MYSQL_GLOBAL_SWITCHOVER_STATE] = marker+ marker.update({+ "state": "repair_required",+ "source_arn": owner["source"].get("DBClusterArn"),+ "target_arn": owner["target"].get("DBClusterArn"),+ "operation_id": owner["operation_id"],+ })+ # The durable in-progress claim already restores as repair-required,+ # so a failed update remains fail-closed across a restart.+ _durably_persist_mysql_global_writer_switch_state(gc, marker)+ return Trueelse:- _mysql_global_writer_switch_owners.pop(id(gc), None)+ gc.pop(_MYSQL_GLOBAL_SWITCHOVER_STATE, None)gc["Status"] = "available"+ if not _durably_persist_mysql_global_writer_switch_state(gc, None):+ owner["state"] = "repair_required"+ gc[_MYSQL_GLOBAL_SWITCHOVER_STATE] = {+ "state": "repair_required",+ "source_arn": owner["source"].get("DBClusterArn"),+ "target_arn": owner["target"].get("DBClusterArn"),+ "operation_id": owner["operation_id"],+ }+ return False+ _mysql_global_writer_switch_owners.pop(id(gc), None)+ return True-def _switch_global_cluster_writer(p, *, allow_data_loss=False):- gc_id = _p(p, "GlobalClusterIdentifier")- target_cluster_id = _p(p, "TargetDbClusterIdentifier")- invalid_id = _invalid_global_cluster_identifier_error(gc_id)- if invalid_id:- return invalid_id- gc = _resolve_global_cluster(gc_id)- if not gc:- return _error("GlobalClusterNotFoundFault",- f"Global cluster {gc_id} not found.", 404)- if not target_cluster_id:- return _error("MissingParameter", "TargetDbClusterIdentifier is required", 400)-- _target_cluster, target_member = _find_global_cluster_target_member(gc, target_cluster_id)- if not target_member:- cluster = _resolve_cluster(target_cluster_id)- if not cluster:- return _error("DBClusterNotFoundFault",- f"DBCluster {target_cluster_id} not found.", 404)- return _error("InvalidGlobalClusterStateFault",- f"DBCluster {target_cluster_id} is not a secondary member of global cluster {gc_id}.", 400)-- members = gc.get("GlobalClusterMembers", [])- current_writer = next((m for m in members if m.get("IsWriter")), None)- if not current_writer or target_member.get("IsWriter") or len(members) < 2:- return _error("InvalidGlobalClusterStateFault",- f"Global cluster {gc_id} does not have a secondary target to promote.", 400)-- _set_global_cluster_writer(gc, target_member)- gc["Status"] = "available"-+def _global_cluster_writer_switch_response(gc, current_writer, target_member, *, allow_data_loss):response_gc = copy.deepcopy(gc)- response_gc["Status"] = "switching-over" if not allow_data_loss else "failing-over"+ response_gc["Status"] = "failing-over" if allow_data_loss else "switching-over"response_gc["FailoverState"] = {"Status": "pending","FromDbClusterArn": current_writer["DBClusterArn"],@@ -7679,8 +8095,571 @@ def _switch_global_cluster_writer(p, *, allow_data_loss=False):return response_gc+def _wait_for_mysql_replication_healthy(replica, source, *, deadline):+ """Wait for both reverse replication threads to reach the expected source."""+ conn = None+ cur = None+ healthy = False+ try:+ conn = _connect_mysql(+ _mysql_replication_connection,+ replica,+ _mysql_deadline_timeout(deadline=deadline),+ )+ if conn is None:+ return False+ cur = conn.cursor()+ while True:+ _execute_mysql(cur, "SHOW REPLICA STATUS", deadline=deadline)+ row = cur.fetchone()+ if isinstance(row, dict):+ status = row+ else:+ description = getattr(cur, "description", None) or ()+ columns = [column[0] for column in description]+ status = dict(zip(columns, row or ()))+ try:+ source_port_matches = int(status.get("Source_Port")) == int(+ source.get("_shared_internal_port"),+ )+ except (TypeError, ValueError):+ source_port_matches = False+ if (+ status.get("Replica_IO_Running") == "Yes"+ and status.get("Replica_SQL_Running") == "Yes"+ and status.get("Source_Host") == source.get("_shared_internal_address")+ and source_port_matches+ ):+ healthy = True+ break+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ break+ time.sleep(min(_MYSQL_REPLICATION_RETRY_INTERVAL, remaining))+ except Exception as e:+ logger.warning(+ "RDS: reverse MySQL replication did not become healthy for %s: %s",+ replica.get("DBClusterIdentifier", "unknown"), e,+ )+ finally:+ if not _close_mysql_resources(+ "replication health check", cursor=cur, connection=conn,+ ):+ healthy = False+ return healthy+++def _rollback_mysql_global_writer_switch(+ owner, *, target_channel_changed=False, reverse_channel_attempted=False,+):+ gc = owner["global_cluster"]+ source = owner["source"]+ target = owner["target"]+ # Compensation has its own bounded budget so safety restoration is tried.+ deadline = time.monotonic() + _MYSQL_GLOBAL_SWITCHOVER_TIMEOUT++ safe = True+ if reverse_channel_attempted:+ try:+ source_reset = bool(_reset_mysql_replication_channel(+ source.get("DBClusterIdentifier", ""),+ source,+ clear_super_read_only=False,+ allow_missing_channel=True,+ deadline=deadline,+ ))+ if source_reset:+ _clear_mysql_replication_metadata(source)+ except Exception as e:+ logger.warning("RDS: reverse channel rollback failed: %s", e)+ source_reset = False+ safe = safe and source_reset+ if target_channel_changed:+ try:+ target_restored = bool(_configure_mysql_replica_from_source(+ target.get("DBClusterIdentifier", ""), target, source,+ deadline=deadline,+ ))+ if target_restored:+ target_restored = _wait_for_mysql_replication_healthy(+ target, source, deadline=deadline,+ )+ except Exception as e:+ logger.warning("RDS: target channel rollback failed: %s", e)+ target_restored = False+ safe = safe and target_restored+ try:+ target_safety = _fence_or_contain_mysql_cluster(target)+ safe = safe and target_safety == "fenced"+ if safe:+ if not _acquire_mysql_lifecycle_lock(deadline):+ safe = False+ else:+ try:+ safe = _mysql_global_writer_switch_still_owned(+ owner, owner["topology"],+ )+ finally:+ _shared_container_lock.release()+ if safe:+ safe = _set_mysql_cluster_writable(source, deadline=deadline)+ except Exception as e:+ logger.warning("RDS: global writer rollback failed: %s", e)+ safe = False+ finally:+ # Repair ownership must be finalized even after compensation expires.+ with _shared_container_lock:+ owned = _active_mysql_global_writer_switch(gc) is owner+ finalized = False+ if owned:+ finalized = _finish_mysql_global_writer_switch(+ owner, repair_required=not safe,+ )+ if safe and owned and not finalized:+ _fence_or_contain_mysql_cluster(source)+ _fence_or_contain_mysql_cluster(target)+ with _shared_container_lock:+ if _active_mysql_global_writer_switch(gc) is owner:+ _finish_mysql_global_writer_switch(+ owner, repair_required=True,+ )+ return safe and owned and finalized+++def _planned_mysql_global_writer_switch_not_ready(target_cluster_id, message=None):+ return _error(+ "InvalidDBClusterStateFault",+ message or f"DBCluster {target_cluster_id} is not ready for a "+ "planned global writer switchover.",+ 400,+ )+++def _repair_committed_mysql_global_writer_switch(+ p,+ gc,+ source,+ target,+ source_member,+ target_member,+ marker,+ original_topology,+ original_identities,+ *,+ deadline,+):+ """Finalize a retry whose durable metadata already names the new writer."""+ gc_id = _p(p, "GlobalClusterIdentifier")+ target_id = _p(p, "TargetDbClusterIdentifier")++ if not _ensure_mysql_control_user(+ source, require_quiescence_grants=True, deadline=deadline,+ ) or not _ensure_mysql_control_user(target, deadline=deadline):+ return _planned_mysql_global_writer_switch_not_ready(target_id)+ if not _ensure_mysql_replication_user(target, deadline=deadline):+ return _planned_mysql_global_writer_switch_not_ready(target_id)+ if any(+ _mysql_writer_fence_active(cluster, deadline=deadline) is not True+ for cluster in (source, target)+ ):+ return _planned_mysql_global_writer_switch_not_ready(target_id)++ if not _acquire_mysql_lifecycle_lock(deadline):+ return _planned_mysql_global_writer_switch_not_ready(target_id)+ try:+ if (+ _resolve_global_cluster(gc_id) is not gc+ or _mysql_global_writer_switch_topology(gc) != original_topology+ or not _mysql_global_writer_switch_repair_ready(+ gc, source, target, marker,+ )+ or original_identities != tuple(+ _mysql_global_writer_switch_identity(cluster)+ for cluster in (source, target)+ )+ or _global_cluster_writer_cluster(gc) is not target+ ):+ return _planned_mysql_global_writer_switch_not_ready(+ target_id,+ f"Global cluster {gc_id} changed while preparing its "+ "committed writer repair.",+ )+ owner = _resume_mysql_global_writer_switch(gc, source, target, marker)+ if owner is None:+ return _planned_mysql_global_writer_switch_not_ready(target_id)+ finally:+ _shared_container_lock.release()++ try:+ if not _configure_mysql_replica_from_source(+ source.get("DBClusterIdentifier", ""), source, target,+ deadline=deadline,+ ):+ raise RuntimeError("committed writer replica repair failed")+ if not _wait_for_mysql_replication_healthy(+ source, target, deadline=deadline,+ ):+ raise RuntimeError("committed writer replica did not become healthy")+ if not _set_mysql_cluster_writable(target, deadline=deadline):+ raise RuntimeError("committed writer could not be made writable")++ source_fenced = _mysql_writer_fence_active(+ source, deadline=deadline,+ ) is True+ target_writable = _mysql_control_query(+ target, "SELECT @@GLOBAL.read_only, @@GLOBAL.super_read_only",+ deadline=deadline,+ ) == ((0, 0),)+ if not _acquire_mysql_lifecycle_lock(deadline):+ raise RuntimeError("committed writer repair lock deadline elapsed")+ try:+ if (+ not source_fenced+ or not target_writable+ or not _mysql_global_writer_switch_still_owned(+ owner, original_topology,+ )+ or _global_cluster_writer_cluster(gc) is not target+ ):+ raise RuntimeError("committed writer repair changed before finalization")+ if not _finish_mysql_global_writer_switch(owner):+ raise RuntimeError("committed writer repair could not be persisted")+ result = _global_cluster_writer_switch_response(+ gc,+ source_member,+ target_member,+ allow_data_loss=False,+ )+ finally:+ _shared_container_lock.release()+ return result+ except Exception as e:+ logger.warning(+ "RDS: committed global writer repair for %s failed: %s", gc_id, e,+ )+ _fence_or_contain_mysql_cluster(source)+ _fence_or_contain_mysql_cluster(target)+ with _shared_container_lock:+ if _active_mysql_global_writer_switch(gc) is owner:+ _finish_mysql_global_writer_switch(owner, repair_required=True)+ return _planned_mysql_global_writer_switch_not_ready(target_id)+++def _planned_mysql_global_writer_switch(p):+ """Synchronously exchange a two-member Aurora MySQL global writer."""+ gc_id = _p(p, "GlobalClusterIdentifier")+ target_cluster_id = _p(p, "TargetDbClusterIdentifier")++ def _not_ready_error(message=None):+ return _planned_mysql_global_writer_switch_not_ready(+ target_cluster_id, message,+ )++ deadline = time.monotonic() + _MYSQL_GLOBAL_SWITCHOVER_TIMEOUT+ if not _acquire_mysql_lifecycle_lock(deadline):+ return _not_ready_error()+ try:+ committed_repair = _resolve_committed_mysql_global_writer_switch_repair(p)+ if committed_repair:+ (+ gc,+ source,+ target,+ current_writer,+ target_member,+ repair_marker,+ ) = committed_repair+ recovering = True+ else:+ resolved, error = _resolve_global_writer_switch(p)+ if error:+ return error+ gc, target, target_member, current_writer = resolved+ source = _resolve_global_member_cluster(current_writer)+ if not source or not target:+ return _not_ready_error()+ repair_marker = _persisted_mysql_global_writer_switch_state(gc)+ recovering = repair_marker is not None+ live_owner = _mysql_global_writer_switch_owners.get(id(gc))+ if live_owner and live_owner.get("global_cluster") is gc:+ return _error(+ "InvalidGlobalClusterStateFault",+ f"Global cluster {gc_id} already has a writer switchover in "+ "progress.",+ 400,+ )+ if recovering and not _mysql_global_writer_switch_repair_matches(+ repair_marker, source, target,+ ):+ return _error(+ "InvalidGlobalClusterStateFault",+ f"Global cluster {gc_id} requires repair by retrying its "+ "interrupted writer switchover target.",+ 400,+ )+ native = tuple(+ _aurora_mysql_8_replication_enabled(cluster)+ for cluster in (source, target)+ )+ if native == (False, False):+ return _switch_global_cluster_writer_locked(+ p, allow_data_loss=False,+ )+ if native != (True, True):+ return _not_ready_error()+ if len(gc.get("GlobalClusterMembers", [])) != 2:+ return _not_ready_error()+ if recovering:+ if not _mysql_global_writer_switch_repair_ready(+ gc, source, target, repair_marker,+ ):+ return _not_ready_error(+ f"Global cluster {gc_id} repair compute is not ready to "+ "retry its interrupted writer switchover.",+ )+ elif not _mysql_global_writer_switch_ready(gc, source, target):+ return _not_ready_error()+ original_topology = _mysql_global_writer_switch_topology(gc)+ original_identities = tuple(+ _mysql_global_writer_switch_identity(cluster)+ for cluster in (source, target)+ )+ finally:+ _shared_container_lock.release()++ if committed_repair:+ return _repair_committed_mysql_global_writer_switch(+ p,+ gc,+ source,+ target,+ current_writer,+ target_member,+ repair_marker,+ original_topology,+ original_identities,+ deadline=deadline,+ )++ def _remaining():+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ raise RuntimeError("global writer switchover deadline elapsed")+ return remaining++ if not _ensure_mysql_control_user(+ source, require_quiescence_grants=True, deadline=deadline,+ ) or not _ensure_mysql_control_user(target, deadline=deadline):+ return _not_ready_error()+ if not _ensure_mysql_replication_user(source, deadline=deadline):+ return _not_ready_error()+ if (+ recovering+ and _mysql_writer_fence_active(source, deadline=deadline) is not True+ ):+ return _not_ready_error()+ if _mysql_writer_fence_active(target, deadline=deadline) is not True:+ return _not_ready_error()++ if not _acquire_mysql_lifecycle_lock(deadline):+ return _not_ready_error()+ try:+ if (+ _resolve_global_cluster(gc_id) is not gc+ or _mysql_global_writer_switch_topology(gc) != original_topology+ or _resolve_global_member_cluster(current_writer) is not source+ or _resolve_global_member_cluster(target_member) is not target+ or not (+ _mysql_global_writer_switch_repair_ready(+ gc, source, target, repair_marker,+ )+ if recovering else+ _mysql_global_writer_switch_ready(gc, source, target)+ )+ or original_identities != tuple(+ _mysql_global_writer_switch_identity(cluster)+ for cluster in (source, target)+ )+ ):+ return _not_ready_error(+ f"Global cluster {gc_id} changed while preparing switchover.",+ )+ owner = (+ _resume_mysql_global_writer_switch(+ gc, source, target, repair_marker,+ )+ if recovering else+ _claim_mysql_global_writer_switch(gc, source, target)+ )+ if owner is None:+ if recovering:+ return _not_ready_error(+ f"Global cluster {gc_id} repair claim changed while "+ "preparing its writer switchover retry.",+ )+ if not _active_mysql_global_writer_switch(gc):+ return _not_ready_error(+ f"Global cluster {gc_id} could not durably record its "+ "planned writer switchover.",+ )+ return _error(+ "InvalidGlobalClusterStateFault",+ f"Global cluster {gc_id} already has a writer switchover in "+ "progress.",+ 400,+ )+ finally:+ _shared_container_lock.release()++ fence_conn = None+ target_channel_changed = False+ reverse_channel_attempted = False+ metadata_flipped = False+ try:+ fence_conn = _mysql_replication_connection(source, timeout=_remaining())+ if fence_conn is None or not _set_mysql_writer_fence(+ fence_conn, deadline=deadline,+ ):+ raise RuntimeError("writer fence failed")+ gtid = _wait_for_mysql_writer_quiescence(+ source, _remaining(), deadline=deadline,+ )+ if gtid is None:+ raise RuntimeError("writer drain failed")+ if not _close_mysql_resources("switchover fence", connection=fence_conn):+ raise RuntimeError("writer fence connection close failed")+ fence_conn = None++ if not _wait_for_mysql_gtid(+ target, gtid, _remaining(), deadline=deadline,+ ):+ raise RuntimeError("target GTID convergence failed")++ target_channel_changed = True+ if not _reset_mysql_replication_channel(+ target.get("DBClusterIdentifier", ""),+ target,+ clear_super_read_only=False,+ allow_missing_channel=recovering,+ deadline=deadline,+ ):+ raise RuntimeError("target replication reset failed")++ reverse_channel_attempted = True+ if not _configure_mysql_replica_from_source(+ source.get("DBClusterIdentifier", ""),+ source,+ target,+ deadline=deadline,+ ):+ raise RuntimeError("reverse replication configuration failed")+ if not _wait_for_mysql_replication_healthy(+ source, target, deadline=deadline,+ ):+ raise RuntimeError("reverse replication did not become healthy")++ if not _acquire_mysql_lifecycle_lock(deadline):+ raise RuntimeError("global writer commit lock deadline elapsed")+ try:+ if not _mysql_global_writer_switch_still_owned(+ owner, owner["topology"],+ ):+ raise RuntimeError("global topology changed before commit")+ _set_global_cluster_writer(gc, target_member)+ metadata_flipped = True+ committed_topology = _mysql_global_writer_switch_topology(gc)+ _clear_mysql_replication_metadata(target)+ _sync_global_mysql_credentials(target, gc)+ finally:+ _shared_container_lock.release()++ if not _set_mysql_cluster_writable(target, deadline=deadline):+ raise RuntimeError("promoted writer could not be made writable")++ source_fenced = _mysql_writer_fence_active(source, deadline=deadline) is True+ target_writable = _mysql_control_query(+ target, "SELECT @@GLOBAL.read_only, @@GLOBAL.super_read_only",+ deadline=deadline,+ ) == ((0, 0),)++ if not _acquire_mysql_lifecycle_lock(deadline):+ raise RuntimeError("global writer finalization lock deadline elapsed")+ try:+ if (+ not source_fenced+ or not target_writable+ or not _mysql_global_writer_switch_still_owned(+ owner, committed_topology,+ )+ or _global_cluster_writer_cluster(gc) is not target+ ):+ raise RuntimeError("global topology changed after commit")+ if not _finish_mysql_global_writer_switch(owner):+ raise RuntimeError(+ "global writer finalization could not be persisted"+ )+ result = _global_cluster_writer_switch_response(+ gc,+ current_writer,+ target_member,+ allow_data_loss=False,+ )+ finally:+ _shared_container_lock.release()+ return result+ except Exception as e:+ logger.warning("RDS: planned global switchover for %s failed: %s", gc_id, e)+ if metadata_flipped:+ _fence_or_contain_mysql_cluster(source)+ _fence_or_contain_mysql_cluster(target)+ with _shared_container_lock:+ if _active_mysql_global_writer_switch(gc) is owner:+ _finish_mysql_global_writer_switch(+ owner, repair_required=True,+ )+ else:+ _rollback_mysql_global_writer_switch(+ owner,+ target_channel_changed=target_channel_changed,+ reverse_channel_attempted=reverse_channel_attempted,+ )+ return _not_ready_error()+ finally:+ _close_mysql_resources("switchover fence", connection=fence_conn)+++def _switch_global_cluster_writer(p, *, allow_data_loss=False):+ with _shared_container_lock:+ return _switch_global_cluster_writer_locked(+ p, allow_data_loss=allow_data_loss,+ )+++def _switch_global_cluster_writer_locked(p, *, allow_data_loss=False):+ resolved, error = _resolve_global_writer_switch(p)+ if error:+ return error+ gc, _target, target_member, current_writer = resolved+ if _active_mysql_global_writer_switch(gc):+ return _error(+ "InvalidGlobalClusterStateFault",+ f"Global cluster {gc['GlobalClusterIdentifier']} already has a "+ "writer switchover in progress.",+ 400,+ )+ _set_global_cluster_writer(gc, target_member)+ gc["Status"] = "available"++ return _global_cluster_writer_switch_response(+ gc,+ current_writer,+ target_member,+ allow_data_loss=allow_data_loss,+ )++def _switchover_global_cluster(p):- result = _switch_global_cluster_writer(p, allow_data_loss=False)+ result = _planned_mysql_global_writer_switch(p)if isinstance(result, tuple):return resultreturn _xml(200, "SwitchoverGlobalClusterResponse",@@ -7704,7 +8683,10 @@ def _failover_global_cluster(p):"AllowDataLoss and Switchover cannot both be specified.",400,)- result = _switch_global_cluster_writer(p, allow_data_loss=allow_data_loss)+ if allow_data_loss:+ result = _switch_global_cluster_writer(p, allow_data_loss=True)+ else:+ result = _planned_mysql_global_writer_switch(p)if isinstance(result, tuple):return resultreturn _xml(200, "FailoverGlobalClusterResponse",diff --git a/tests/test_rds.py b/tests/test_rds.pyindex 6a0c2520..291546a1 100644--- a/tests/test_rds.py+++ b/tests/test_rds.py@@ -5052,6 +5052,8 @@ def test_rds_restore_state_respawns_docker_container(monkeypatch):"pending-password-rotation","global-secondary","global-secondary-before-control-user",+ "repair-required",+ "repair-unfenced","last-member-deleted-during-readiness","last-members-deleted-before-start",],@@ -5072,6 +5074,7 @@ def test_rds_restore_state_respawns_one_container_per_cluster(rotations = []grants = []replication_configs = []+ forced_repair_removals = []remaining_legacy_names = set()legacy_owner_by_name = {}writer_legacy_container_name = [None]@@ -5222,6 +5225,22 @@ def test_rds_restore_state_respawns_one_container_per_cluster("_configure_or_defer_mysql_replication",_configure_replication,)+ if scenario.startswith("repair-"):+ monkeypatch.setattr(+ m,+ "_mysql_writer_fence_active",+ lambda _cluster: scenario == "repair-required",+ )+ if scenario == "repair-unfenced":+ monkeypatch.setattr(+ m, "_stop_cluster_shared_container", lambda *_args: False,+ )+ monkeypatch.setattr(+ m,+ "_force_remove_cluster_compute_preserving_volume",+ lambda cluster_id, _cluster: forced_repair_removals.append(cluster_id)+ or True,+ )if scenario.startswith("global-secondary"):monkeypatch.setattr(m, "_mysql_replication_secondary", lambda _cluster: True)@@ -5280,6 +5299,15 @@ def test_rds_restore_state_respawns_one_container_per_cluster(cluster_record["_mysql_gtid_initialized_at_creation"] = Trueif scenario == "global-secondary":cluster_record["_mysql_control_user_ready"] = True+ elif scenario.startswith("repair-"):+ cluster_record.update({+ "DBClusterArn": (+ "arn:aws:rds:us-east-1:111111111111:cluster:"+ "restored-shared-cluster"+ ),+ "GlobalClusterIdentifier": "repair-global",+ "_mysql_gtid_initialized_at_creation": True,+ })clusters.set_scoped(account_id, region, cluster_id, cluster_record)instances = AccountRegionScopedDict()for db_id in ("restored-writer", "restored-reader"):@@ -5304,6 +5332,20 @@ def test_rds_restore_state_respawns_one_container_per_cluster(m._instances.clear()m._clusters.clear()try:+ global_clusters = {}+ if scenario.startswith("repair-"):+ global_clusters = {+ "repair-global": {+ "GlobalClusterIdentifier": "repair-global",+ "GlobalClusterMembers": [{+ "DBClusterArn": cluster_record["DBClusterArn"],+ "IsWriter": True,+ }],+ m._MYSQL_GLOBAL_SWITCHOVER_STATE: {+ "state": "repair_required",+ },+ },+ }m.restore_state({"instances": instances,"clusters": clusters,@@ -5313,7 +5355,7 @@ def test_rds_restore_state_respawns_one_container_per_cluster("db_cluster_param_groups": {},"db_cluster_snapshots": {},"option_groups": {},- "global_clusters": {},+ "global_clusters": global_clusters,"tags": {},"port_counter": 16010,})@@ -5429,6 +5471,23 @@ def test_rds_restore_state_respawns_one_container_per_cluster(assert removed_volumes == []return+ if scenario.startswith("repair-"):+ restored_cluster = m._clusters.get(cluster_id)+ assert {"--read-only=ON", "--super-read-only=ON"} <= set(+ runs[0]["command"],+ )+ assert restored_cluster["_shared_container_ready"] is False+ assert all(+ m._instances.get(db_id)["DBInstanceStatus"] == "failed"+ for db_id in ("restored-writer", "restored-reader")+ )+ assert replication_configs == []+ assert removed_volumes == []+ assert forced_repair_removals == (+ [cluster_id] if scenario == "repair-unfenced" else []+ )+ return+assert len(runs) == 1assert runs[0]["name"] == m._rds_cluster_docker_name(cluster_id)if scenario.startswith("global-secondary"):@@ -6568,17 +6627,10 @@ def _delete_global_cluster(client, global_id):def _cleanup_two_member_global(east, global_id, primary_arn=None, secondary_arn=None):- if primary_arn:- try:- east.switchover_global_cluster(- GlobalClusterIdentifier=global_id,- TargetDbClusterIdentifier=primary_arn,- )- except ClientError:- pass- for cluster_arn in (secondary_arn, primary_arn):- if cluster_arn:- _remove_global_member(east, global_id, cluster_arn)+ for _attempt in range(2):+ for cluster_arn in (secondary_arn, primary_arn):+ if cluster_arn:+ _remove_global_member(east, global_id, cluster_arn)_delete_global_cluster(east, global_id)@@ -7075,7 +7127,7 @@ def test_aurora_global_metadata_spans_regions():_delete_cluster(east, primary_id)-def test_switchover_global_cluster_promotes_foreign_region_member_arn():+def test_switchover_global_cluster_rejects_headless_foreign_region_target():east = _regional_rds("us-east-1")west = _regional_rds("us-west-2")suffix = uuid.uuid4().hex[:8]@@ -7106,39 +7158,21 @@ def test_switchover_global_cluster_promotes_foreign_region_member_arn():)["DBCluster"]secondary_arn = secondary["DBClusterArn"]- response = east.switchover_global_cluster(- GlobalClusterIdentifier=global_id,- TargetDbClusterIdentifier=secondary_arn,- )["GlobalCluster"]- assert response["Status"] == "switching-over"- assert response["FailoverState"]["Status"] == "pending"- assert response["FailoverState"]["FromDbClusterArn"] == primary_arn- assert response["FailoverState"]["ToDbClusterArn"] == secondary_arn- assert response["FailoverState"]["IsDataLossAllowed"] is False-- members = {m["DBClusterArn"]: m for m in response["GlobalClusterMembers"]}- assert members[primary_arn]["IsWriter"] is False- assert members[secondary_arn]["IsWriter"] is True- assert members[secondary_arn]["Readers"] == [primary_arn]+ with pytest.raises(ClientError) as exc:+ east.switchover_global_cluster(+ GlobalClusterIdentifier=global_id,+ TargetDbClusterIdentifier=secondary_arn,+ )+ assert exc.value.response["Error"]["Code"] == "InvalidDBClusterStateFault"- final = west.describe_global_clusters(+ final = east.describe_global_clusters(GlobalClusterIdentifier=global_id,)["GlobalClusters"][0]final_members = {m["DBClusterArn"]: m for m in final["GlobalClusterMembers"]}assert final["Status"] == "available"assert "FailoverState" not in final- assert final_members[primary_arn]["IsWriter"] is False- assert final_members[secondary_arn]["IsWriter"] is True-- switchback = west.switchover_global_cluster(- GlobalClusterIdentifier=global_id,- TargetDbClusterIdentifier=primary_arn,- )["GlobalCluster"]- switchback_members = {- m["DBClusterArn"]: m for m in switchback["GlobalClusterMembers"]- }- assert switchback_members[primary_arn]["IsWriter"] is True- assert switchback_members[secondary_arn]["IsWriter"] is False+ assert final_members[primary_arn]["IsWriter"] is True+ assert final_members[secondary_arn]["IsWriter"] is Falsefinally:_cleanup_two_member_global(east, global_id, primary_arn, secondary_arn)_delete_cluster(west, secondary_id)@@ -10530,6 +10564,182 @@ def test_rds_mysql_quiescence_epoch_change_suppresses_result(monkeypatch):assert m._wait_for_mysql_writer_quiescence(cluster, 1) is None+def test_rds_mysql_writer_switch_state_round_trip_requires_repair():+ from ministack.services import rds as m++ gc_id = "persisted-switch"+ global_cluster = {+ "GlobalClusterIdentifier": gc_id,+ "GlobalClusterMembers": [],+ m._MYSQL_GLOBAL_SWITCHOVER_STATE: {+ "state": "in_progress",+ "source_arn": "source-arn",+ "target_arn": "target-arn",+ },+ }+ m._global_clusters.clear()+ m._global_clusters[gc_id] = global_cluster+ try:+ persisted = m.get_state()+ m._global_clusters.clear()++ m.restore_state({+ "instances": {},+ "clusters": {},+ "global_clusters": persisted["global_clusters"],+ })++ restored = m._global_clusters[gc_id]+ marker = restored[m._MYSQL_GLOBAL_SWITCHOVER_STATE]+ assert marker == {+ "state": "repair_required",+ "source_arn": "source-arn",+ "target_arn": "target-arn",+ }+ assert m._active_mysql_global_writer_switch(restored) is marker+ finally:+ m._global_clusters.clear()+++def test_rds_repair_containment_does_not_treat_docker_error_as_absent(+ monkeypatch,+):+ from ministack.services import rds as m++ class FailingContainers:+ def get(self, _identifier):+ raise TimeoutError("Docker daemon timed out")++ class FakeDocker:+ def __init__(self):+ self.containers = FailingContainers()++ cluster = {+ "DBClusterArn": (+ "arn:aws:rds:us-west-2:111111111111:cluster:repair-cluster"+ ),+ "_shared_container_id": "possibly-running-container",+ }+ monkeypatch.setattr(m, "_get_docker", lambda: FakeDocker())++ assert m._force_remove_cluster_compute_preserving_volume(+ "repair-cluster", cluster,+ ) is False+ assert cluster["_shared_container_id"] == "possibly-running-container"+++def test_rds_repair_containment_uses_cluster_arn_scope(monkeypatch):+ from ministack.services import rds as m++ removed = []++ class NotFound(Exception):+ status_code = 404++ class Container:+ id = "target-container"+ labels = {+ "ministack": "rds",+ "cluster_id": "repair-cluster",+ "account_id": "111111111111",+ "region": "us-west-2",+ }++ def remove(self, **kwargs):+ removed.append(kwargs)++ container = Container()+ target_name = m._rds_cluster_docker_name(+ "repair-cluster", "111111111111", "us-west-2",+ )++ class Containers:+ def get(self, identifier):+ if not removed and identifier in (container.id, target_name):+ return container+ raise NotFound()++ class FakeDocker:+ def __init__(self):+ self.containers = Containers()++ cluster = {+ "DBClusterArn": (+ "arn:aws:rds:us-west-2:111111111111:cluster:repair-cluster"+ ),+ "_shared_container_id": container.id,+ "_shared_internal_address": "172.20.0.20",+ "_shared_internal_port": 3306,+ }+ monkeypatch.setattr(m, "_get_docker", lambda: FakeDocker())++ assert m._contain_cluster_compute_preserving_volume(+ "repair-cluster", cluster,+ ) is True+ assert removed == [{"force": True, "v": False}]+ assert cluster["_shared_container_id"] is None+ assert cluster["_shared_internal_address"] is None+ assert cluster["_shared_internal_port"] is None+++def test_rds_member_status_update_uses_cluster_arn_scope():+ from ministack.services import rds as m++ instance_id = "same-name-member"+ east = {"DBInstanceIdentifier": instance_id, "DBInstanceStatus": "available"}+ west = {"DBInstanceIdentifier": instance_id, "DBInstanceStatus": "available"}+ m._instances.clear()+ try:+ m._instances.set_scoped(+ "111111111111", "us-east-1", instance_id, east,+ )+ m._instances.set_scoped(+ "111111111111", "us-west-2", instance_id, west,+ )+ cluster = {+ "DBClusterArn": (+ "arn:aws:rds:us-west-2:111111111111:cluster:repair-cluster"+ ),+ "DBClusterMembers": [{"DBInstanceIdentifier": instance_id}],+ }++ m._set_cluster_members_status(cluster, "failed")++ assert east["DBInstanceStatus"] == "available"+ assert west["DBInstanceStatus"] == "failed"+ finally:+ m._instances.clear()+++def test_rds_cluster_refence_uses_one_total_deadline(monkeypatch):+ from ministack.services import rds as m++ class Connection:+ def close(self):+ pass++ deadlines = []+ monkeypatch.setattr(m.time, "monotonic", lambda: 100.0)+ monkeypatch.setattr(+ m,+ "_mysql_replication_connection",+ lambda _cluster, timeout: Connection() if timeout == 3 else None,+ )+ monkeypatch.setattr(+ m,+ "_set_mysql_writer_fence",+ lambda _conn, *, deadline: deadlines.append(deadline) or True,+ )+ monkeypatch.setattr(+ m,+ "_mysql_writer_fence_active",+ lambda _cluster, *, deadline: deadlines.append(deadline) or True,+ )++ assert m._set_mysql_cluster_fenced({}, timeout=3) is True+ assert deadlines == [103.0, 103.0]++def _mysql_switchover_unit_topology(monkeypatch, m):topology = _mysql_replication_unit_topology()writer, target, writer_member, target_member, global_cluster = topology@@ -10569,6 +10779,9 @@ def _mysql_switchover_unit_topology(monkeypatch, m):}] if cluster.get("DBClusterMembers") else [],)monkeypatch.setattr(m, "_mysql_global_writer_switch_owners", {})+ monkeypatch.setattr(+ m, "_wait_for_mysql_replication_healthy", lambda *_a, **_k: True,+ )return topology@@ -10579,106 +10792,1072 @@ def _mysql_switchover_params(target="secondary"):}-def test_rds_mysql_gtid_wait_recomputes_shared_deadline(monkeypatch):+def test_rds_mysql_switchover_claim_is_durable_when_persistence_enabled(+ monkeypatch,+ tmp_path,+):from ministack.services import rds as m- now = [100.0]- observed = {}- monkeypatch.setattr(m.time, "monotonic", lambda: now[0])-- class Cursor:- def __init__(self, connection):- self.connection = connection-- def execute(self, statement, params):- observed["sql"] = (statement, params)-- def fetchone(self):- return (0,)+ writer, target, _writer_member, target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )+ m._global_clusters.clear()+ m._global_clusters["global-repl"] = global_cluster+ monkeypatch.setattr(m.persistence, "PERSIST_STATE", True)+ monkeypatch.setattr(m.persistence, "STATE_DIR", str(tmp_path))+ try:+ owner = m._claim_mysql_global_writer_switch(+ global_cluster, writer, target,+ )- def close(self):- pass+ assert owner is not None+ marker = global_cluster[m._MYSQL_GLOBAL_SWITCHOVER_STATE]+ assert marker["operation_id"]+ persisted = m.persistence.load_state("rds")+ assert m._persisted_mysql_global_writer_switch_marker(+ persisted, global_cluster,+ ) == marker++ m._set_global_cluster_writer(global_cluster, target_member)+ assert m._finish_mysql_global_writer_switch(owner) is True+ persisted = m.persistence.load_state("rds")+ persisted_gc = m._persisted_mysql_global_writer_switch_cluster(+ persisted, global_cluster,+ )+ assert persisted_gc is not None+ assert m._MYSQL_GLOBAL_SWITCHOVER_STATE not in persisted_gc+ assert next(+ member+ for member in persisted_gc["GlobalClusterMembers"]+ if member["DBClusterArn"] == target["DBClusterArn"]+ )["IsWriter"] is True+ finally:+ m._mysql_global_writer_switch_owners.clear()+ m._global_clusters.clear()- class Connection:- def cursor(self):- now[0] += 4- return Cursor(self)- def close(self):- pass+def test_rds_mysql_switchover_claim_fails_closed_when_persistence_fails(+ monkeypatch,+):+ from ministack.services import rds as m- def connect(_cluster, *, timeout):- observed["connect_timeout"] = timeout- return Connection()+ writer, target, _writer_member, _target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )+ monkeypatch.setattr(m.persistence, "PERSIST_STATE", True)+ monkeypatch.setattr(m.persistence, "save_state", lambda *_args: None)+ monkeypatch.setattr(m.persistence, "load_state", lambda *_args: None)- monkeypatch.setattr(m, "_mysql_replication_connection", connect)+ owner = m._claim_mysql_global_writer_switch(global_cluster, writer, target)- assert m._wait_for_mysql_gtid(- {"DBClusterIdentifier": "secondary"}, "source:1-9", 30,- deadline=130.0,- ) is True- assert observed["connect_timeout"] == 30.0- assert observed["sql"][1] == ("source:1-9", 26.0)+ assert owner is None+ assert m._MYSQL_GLOBAL_SWITCHOVER_STATE not in global_cluster+ assert m._mysql_global_writer_switch_owners == {}[email protected]("mutation", ["password", "topology"])-def test_rds_active_switchover_rejects_mutation(monkeypatch, mutation):+def test_rds_mysql_switchover_finalization_failure_retains_repair_state(+ monkeypatch,+):from ministack.services import rds as m- writer, target, writer_member, target_member, global_cluster = (+ writer, target, _writer_member, _target_member, global_cluster = (_mysql_switchover_unit_topology(monkeypatch, m))- with m._shared_container_lock:- owner = m._claim_mysql_global_writer_switch(global_cluster, writer, target)- if mutation == "password":- result = m._modify_db_cluster({- "DBClusterIdentifier": "primary",- "MasterUserPassword": "changed-password",- })- elif mutation == "topology":- result = m._remove_from_global_cluster({- "GlobalClusterIdentifier": "global-repl",- "DbClusterIdentifier": "secondary",- })- else:- result = m._failover_global_cluster({- "GlobalClusterIdentifier": "global-repl",- "TargetDbClusterIdentifier": "secondary",- "AllowDataLoss": "true",- })+ persist_results = iter((True, False))+ monkeypatch.setattr(+ m,+ "_durably_persist_mysql_global_writer_switch_state",+ lambda _gc, _marker: next(persist_results),+ )- assert result[0] == 400+ owner = m._claim_mysql_global_writer_switch(global_cluster, writer, target)++ assert owner is not None+ assert m._finish_mysql_global_writer_switch(owner) is Falseassert m._active_mysql_global_writer_switch(global_cluster) is owner- assert writer_member["IsWriter"] is True- assert target_member["IsWriter"] is False- assert writer.get("_MasterUserPassword") != "changed-password"- assert target["GlobalClusterIdentifier"] == "global-repl"- assert target["_mysql_replication_source_arn"] == writer["DBClusterArn"]+ assert global_cluster[m._MYSQL_GLOBAL_SWITCHOVER_STATE] == {+ "state": "repair_required",+ "source_arn": writer["DBClusterArn"],+ "target_arn": target["DBClusterArn"],+ "operation_id": owner["operation_id"],+ }@pytest.mark.parametrize(- ("handler_name", "impl_name"),+ "invalid",[- ("_create_db_instance", "_create_db_instance_impl"),- ("_delete_db_instance", "_delete_db_instance_impl"),+ "current-writer", "third-member", "engine", "version", "storage",+ "headless", "unready", "status", "member",],)-def test_rds_instance_mutation_prevents_switchover_claim(+def test_rds_switchover_requires_ready_two_member_mysql8_topology(monkeypatch,- handler_name,- impl_name,+ invalid,):from ministack.services import rds as m- writer, target, _writer_member, _target_member, global_cluster = (+ writer, target, writer_member, target_member, global_cluster = (_mysql_switchover_unit_topology(monkeypatch, m))- entered = threading.Event()- release = threading.Event()- claimed = threading.Event()- order = []+ target_id = "secondary"+ if invalid == "current-writer":+ writer_member["IsWriter"] = False+ target_member["IsWriter"] = True+ target_id = "secondary"+ elif invalid == "third-member":+ global_cluster["GlobalClusterMembers"].append({+ "DBClusterArn": "arn:aws:rds:eu-west-1:111111111111:cluster:third",+ "IsWriter": False,+ })+ elif invalid == "engine":+ target["Engine"] = "aurora-postgresql"+ elif invalid == "version":+ target["EngineVersion"] = "8.0.mysql_aurora.3.11.1"+ elif invalid == "storage":+ target["_shared_storage_initialized"] = False+ elif invalid == "headless":+ target["DBClusterMembers"] = []+ elif invalid == "unready":+ target["_shared_container_ready"] = False+ elif invalid == "status":+ target["Status"] = "creating"+ else:+ monkeypatch.setattr(+ m, "_cluster_member_instances",+ lambda cluster: [{"DBInstanceStatus": "stopped"}]+ if cluster is target else [{"DBInstanceStatus": "available"}],+ )- def mutation_impl(_params):+ monkeypatch.setattr(+ m, "_ensure_mysql_control_user",+ lambda *_args, **_kwargs: pytest.fail("invalid topology reached MySQL"),+ )+ result = m._planned_mysql_global_writer_switch(+ _mysql_switchover_params(target_id),+ )+ assert result[0] == 400+ assert writer_member["IsWriter"] is (invalid != "current-writer")+ assert target_member["IsWriter"] is (invalid == "current-writer")+++def test_rds_switchover_preserves_non_mysql_multi_member_metadata_behavior(+ monkeypatch,+):+ from ministack.services import rds as m++ writer, target, writer_member, target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )+ writer["Engine"] = target["Engine"] = "aurora-postgresql"+ global_cluster["Engine"] = "aurora-postgresql"+ global_cluster["GlobalClusterMembers"].append({+ "DBClusterArn": "arn:aws:rds:eu-west-1:111111111111:cluster:third",+ "IsWriter": False,+ })++ result = m._planned_mysql_global_writer_switch(_mysql_switchover_params())++ assert result["Status"] == "switching-over"+ assert writer_member["IsWriter"] is False+ assert target_member["IsWriter"] is True+++ "mode",+ [{}, {"Switchover": "true"}, {"AllowDataLoss": "false"}],+)+def test_rds_failover_lossless_modes_use_planned_switchover(monkeypatch, mode):+ from ministack.services import rds as m++ *_, global_cluster = _mysql_switchover_unit_topology(monkeypatch, m)+ calls = []+ monkeypatch.setattr(+ m,+ "_planned_mysql_global_writer_switch",+ lambda params: calls.append(params) or global_cluster,+ )+ monkeypatch.setattr(+ m,+ "_switch_global_cluster_writer",+ lambda *_args, **_kwargs: pytest.fail("lossless failover used metadata path"),+ )+ params = _mysql_switchover_params() | mode++ status, _headers, _body = m._failover_global_cluster(params)++ assert status == 200+ assert calls == [params]+++def test_rds_switchover_orders_lossless_data_plane_before_metadata(monkeypatch):+ from ministack.services import rds as m++ writer, target, writer_member, target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )+ events = []++ class FenceConnection:+ def close(self):+ events.append("fence-close")++ monkeypatch.setattr(+ m, "_ensure_mysql_control_user",+ lambda cluster, **kwargs: events.append(+ ("control", cluster, kwargs.get("require_quiescence_grants", False)),+ ) or True,+ )+ monkeypatch.setattr(+ m, "_ensure_mysql_replication_user",+ lambda cluster, **_kwargs: events.append(("replication-user", cluster)) or True,+ )+ monkeypatch.setattr(+ m, "_mysql_writer_fence_active",+ lambda cluster, **_kwargs: events.append(("target-fenced", cluster)) or True,+ )+ monkeypatch.setattr(+ m,+ "_durably_persist_mysql_global_writer_switch_state",+ lambda _gc, marker: events.append(+ "persist-claim" if marker else "persist-final",+ ) or True,+ )+ monkeypatch.setattr(+ m, "_mysql_replication_connection",+ lambda cluster, **_kwargs: events.append(("fence-connect", cluster))+ or FenceConnection(),+ )+ monkeypatch.setattr(+ m, "_set_mysql_writer_fence",+ lambda _conn, **_kwargs: events.append("source-fence") or True,+ )+ monkeypatch.setattr(+ m, "_wait_for_mysql_writer_quiescence",+ lambda cluster, _timeout, **_kwargs: events.append(("drain", cluster)) or "source:1-9",+ )+ monkeypatch.setattr(+ m, "_wait_for_mysql_gtid",+ lambda cluster, gtid, _timeout, **_kwargs: events.append(("gtid", cluster, gtid)) or True,+ )+ monkeypatch.setattr(+ m, "_reset_mysql_replication_channel",+ lambda cluster_id, cluster, **kwargs: events.append(+ ("reset", cluster_id, cluster, kwargs["clear_super_read_only"]),+ ) or True,+ )++ def configure(cluster_id, replica, source, **_kwargs):+ assert writer_member["IsWriter"] is True+ assert target_member["IsWriter"] is False+ events.append(("configure", cluster_id, replica, source))+ replica["_mysql_replication_source_arn"] = source["DBClusterArn"]+ return True++ def healthy(replica, source, **_kwargs):+ events.append(("healthy", replica, source))+ return True++ def writable(cluster, **_kwargs):+ assert writer_member["IsWriter"] is False+ assert target_member["IsWriter"] is True+ events.append(("writable", cluster))+ return True++ monkeypatch.setattr(m, "_configure_mysql_replica_from_source", configure)+ monkeypatch.setattr(m, "_wait_for_mysql_replication_healthy", healthy)+ monkeypatch.setattr(m, "_set_mysql_cluster_writable", writable)+ monkeypatch.setattr(+ m, "_mysql_control_query",+ lambda cluster, *_args, **_kwargs: ((0, 0),) if cluster is target else None,+ )+ monkeypatch.setattr(m, "_sync_global_mysql_credentials", lambda *_args: None)+ response_lock_held = []+ original_response = m._global_cluster_writer_switch_response++ def response_under_lock(*args, **kwargs):+ acquired = []++ def try_lock():+ locked = m._shared_container_lock.acquire(blocking=False)+ acquired.append(locked)+ if locked:+ m._shared_container_lock.release()++ contender = threading.Thread(target=try_lock)+ contender.start()+ contender.join(timeout=1)+ assert not contender.is_alive()+ response_lock_held.append(not acquired[0])+ return original_response(*args, **kwargs)++ monkeypatch.setattr(+ m, "_global_cluster_writer_switch_response", response_under_lock,+ )++ status, _headers, body = m._switchover_global_cluster(+ _mysql_switchover_params(),+ )+ body = body.decode()++ assert status == 200+ assert "<Status>switching-over</Status>" in body+ assert f"<FromDbClusterArn>{writer['DBClusterArn']}</FromDbClusterArn>" in body+ assert f"<ToDbClusterArn>{target['DBClusterArn']}</ToDbClusterArn>" in body+ assert "<IsDataLossAllowed>false</IsDataLossAllowed>" in body+ assert global_cluster["Status"] == "available"+ assert writer_member["IsWriter"] is False+ assert target_member["IsWriter"] is True+ assert target_member["Readers"] == [writer["DBClusterArn"]]+ assert m._mysql_global_writer_switch_owners == {}+ assert m._MYSQL_GLOBAL_SWITCHOVER_STATE not in global_cluster+ assert response_lock_held == [True]+ assert [event[0] if isinstance(event, tuple) else event for event in events] == [+ "control", "control", "replication-user", "target-fenced",+ "persist-claim", "fence-connect", "source-fence", "drain",+ "fence-close", "gtid", "reset", "configure", "healthy", "writable",+ "target-fenced", "persist-final",+ ]+ assert ("reset", "secondary", target, False) in events+ assert ("configure", "primary", writer, target) in events++[email protected]("committed", [False, True])+def test_rds_switchover_retry_recovers_restored_repair_marker(+ monkeypatch,+ committed,+):+ from ministack.services import rds as m++ writer, target, writer_member, target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )+ operation_id = "interrupted-operation"+ if committed:+ m._set_global_cluster_writer(global_cluster, target_member)+ writer["_mysql_replication_source_arn"] = target["DBClusterArn"]+ target.pop("_mysql_replication_source_arn", None)+ global_cluster[m._MYSQL_GLOBAL_SWITCHOVER_STATE] = {+ "state": "repair_required",+ "source_arn": writer["DBClusterArn"],+ "target_arn": target["DBClusterArn"],+ "operation_id": operation_id,+ }+ for cluster in (writer, target):+ cluster["_shared_container_ready"] = False+ cluster["Status"] = "creating"++ class FenceConnection:+ def close(self):+ pass++ monkeypatch.setattr(m, "_ensure_mysql_control_user", lambda *_a, **_k: True)+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)+ monkeypatch.setattr(m, "_mysql_writer_fence_active", lambda *_a, **_k: True)+ monkeypatch.setattr(+ m, "_mysql_replication_connection", lambda *_a, **_k: FenceConnection(),+ )+ monkeypatch.setattr(m, "_set_mysql_writer_fence", lambda *_a, **_k: True)+ monkeypatch.setattr(+ m, "_wait_for_mysql_writer_quiescence",+ lambda *_a, **_k: "source:1-9",+ )+ monkeypatch.setattr(m, "_wait_for_mysql_gtid", lambda *_a, **_k: True)++ reset_calls = []+ monkeypatch.setattr(+ m,+ "_reset_mysql_replication_channel",+ lambda *_a, **kwargs: reset_calls.append(kwargs) or True,+ )++ def configure(_cluster_id, replica, source, **_kwargs):+ replica["_mysql_replication_source_arn"] = source["DBClusterArn"]+ return True++ monkeypatch.setattr(m, "_configure_mysql_replica_from_source", configure)+ monkeypatch.setattr(m, "_wait_for_mysql_replication_healthy", lambda *_a, **_k: True)+ monkeypatch.setattr(m, "_set_mysql_cluster_writable", lambda *_a, **_k: True)+ monkeypatch.setattr(m, "_mysql_control_query", lambda *_a, **_k: ((0, 0),))+ monkeypatch.setattr(m, "_sync_global_mysql_credentials", lambda *_a: None)+ monkeypatch.setattr(+ m,+ "_durably_persist_mysql_global_writer_switch_state",+ lambda _gc, expected: expected is None,+ )++ result = m._planned_mysql_global_writer_switch(_mysql_switchover_params())++ assert isinstance(result, dict)+ assert writer_member["IsWriter"] is False+ assert target_member["IsWriter"] is True+ assert m._MYSQL_GLOBAL_SWITCHOVER_STATE not in global_cluster+ assert m._mysql_global_writer_switch_owners == {}+ if committed:+ assert reset_calls == []+ else:+ assert reset_calls and reset_calls[0]["allow_missing_channel"] is True+++ "failure",+ ["drain", "gtid", "target-reset", "reverse", "reverse-health", "commit-gate"],+)+def test_rds_switchover_failure_restores_original_topology(monkeypatch, failure):+ from ministack.services import rds as m++ writer, target, writer_member, target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )+ events = []++ class FenceConnection:+ def close(self):+ events.append("fence-close")++ monkeypatch.setattr(m, "_ensure_mysql_control_user", lambda *_a, **_k: True)+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)+ monkeypatch.setattr(m, "_mysql_writer_fence_active", lambda _cluster, **_kwargs: True)+ monkeypatch.setattr(m, "_mysql_replication_connection", lambda *_a, **_k: FenceConnection())+ monkeypatch.setattr(m, "_set_mysql_writer_fence", lambda _conn, **_kwargs: True)+ monkeypatch.setattr(+ m, "_wait_for_mysql_writer_quiescence",+ lambda *_args, **_kwargs: None if failure == "drain" else "source:1-9",+ )+ monkeypatch.setattr(+ m, "_wait_for_mysql_gtid",+ lambda *_args, **_kwargs: failure != "gtid",+ )++ def reset(cluster_id, cluster, **kwargs):+ events.append((+ "reset", cluster_id, cluster,+ kwargs.get("allow_missing_channel", False),+ ))+ if failure == "target-reset" and cluster is target:+ raise RuntimeError("target reset failed")+ return True++ def configure(cluster_id, replica, source, **_kwargs):+ events.append(("configure", cluster_id, replica, source))+ if failure == "reverse" and replica is writer and source is target:+ return False+ if failure == "commit-gate" and replica is writer:+ target["_shared_container_ready"] = False+ elif failure == "commit-gate" and replica is target:+ target["_shared_container_ready"] = True+ replica["_mysql_replication_source_arn"] = source["DBClusterArn"]+ return True++ monkeypatch.setattr(m, "_reset_mysql_replication_channel", reset)+ monkeypatch.setattr(m, "_configure_mysql_replica_from_source", configure)++ def healthy(replica, source, **_kwargs):+ events.append(("healthy", replica, source))+ return not (failure == "reverse-health" and replica is writer)++ monkeypatch.setattr(m, "_wait_for_mysql_replication_healthy", healthy)+ monkeypatch.setattr(+ m, "_set_mysql_cluster_writable",+ lambda cluster, **_kwargs: events.append(("writable", cluster)) or True,+ )++ result = m._planned_mysql_global_writer_switch(_mysql_switchover_params())++ assert result[0] == 400+ assert writer_member["IsWriter"] is True+ assert target_member["IsWriter"] is False+ assert global_cluster["Status"] == "available"+ assert m._mysql_global_writer_switch_owners == {}+ assert [event for event in events if isinstance(event, tuple)][-1] == (+ "writable", writer,+ )+ if failure in ("target-reset", "reverse", "reverse-health", "commit-gate"):+ assert ("configure", "secondary", target, writer) in events+ if failure in ("reverse", "reverse-health", "commit-gate"):+ configure_index = events.index(("configure", "secondary", target, writer))+ assert events[configure_index - 1] == ("reset", "primary", writer, True)+ if failure == "reverse-health":+ assert [event for event in events if event[0] == "healthy"] == [+ ("healthy", writer, target),+ ("healthy", target, writer),+ ]+++ "failure", ["write-enable", "source-fence", "persist-final"],+)+def test_rds_switchover_post_commit_failure_secures_both_members(monkeypatch, failure):+ from ministack.services import rds as m++ writer, target, writer_member, target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )++ class FenceConnection:+ def close(self):+ pass++ monkeypatch.setattr(m, "_ensure_mysql_control_user", lambda *_a, **_k: True)+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)+ monkeypatch.setattr(+ m,+ "_mysql_writer_fence_active",+ lambda cluster, **_kwargs: not (+ failure == "source-fence" and cluster is writer+ ),+ )+ monkeypatch.setattr(m, "_mysql_replication_connection", lambda *_a, **_k: FenceConnection())+ monkeypatch.setattr(m, "_set_mysql_writer_fence", lambda _conn, **_kwargs: True)+ monkeypatch.setattr(m, "_wait_for_mysql_writer_quiescence", lambda *_a, **_k: "source:1-9")+ monkeypatch.setattr(m, "_wait_for_mysql_gtid", lambda *_a, **_k: True)+ monkeypatch.setattr(m, "_reset_mysql_replication_channel", lambda *_a, **_k: True)+ monkeypatch.setattr(+ m, "_configure_mysql_replica_from_source", lambda *_a, **_k: True,+ )+ write_attempts = []+ monkeypatch.setattr(+ m, "_set_mysql_cluster_writable",+ lambda cluster, **_kwargs: write_attempts.append(cluster)+ or failure != "write-enable",+ )+ secured = []+ monkeypatch.setattr(+ m,+ "_fence_or_contain_mysql_cluster",+ lambda cluster: secured.append(cluster) or "fenced",+ )+ monkeypatch.setattr(m, "_sync_global_mysql_credentials", lambda *_args: None)+ monkeypatch.setattr(+ m,+ "_durably_persist_mysql_global_writer_switch_state",+ lambda _gc, marker: not (failure == "persist-final" and marker is None),+ )++ result = m._planned_mysql_global_writer_switch(_mysql_switchover_params())++ assert result[0] == 400+ assert writer_member["IsWriter"] is False+ assert target_member["IsWriter"] is True+ assert write_attempts == [target]+ assert secured == [writer, target]+ owner = m._active_mysql_global_writer_switch(global_cluster)+ assert owner is not None and owner["state"] == "repair_required"+ assert global_cluster[m._MYSQL_GLOBAL_SWITCHOVER_STATE]["state"] == (+ "repair_required"+ )+ assert global_cluster["Status"] == "available"+ assert b"<Status>switching-over</Status>" in m._describe_global_clusters({+ "GlobalClusterIdentifier": "global-repl",+ })[2]+++def test_rds_switchover_sync_failure_keeps_committed_topology_fenced(monkeypatch):+ from ministack.services import rds as m++ writer, target, writer_member, target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )+ events = []++ class FenceConnection:+ def close(self):+ pass++ monkeypatch.setattr(m, "_ensure_mysql_control_user", lambda *_a, **_k: True)+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)+ monkeypatch.setattr(m, "_mysql_writer_fence_active", lambda *_a, **_k: True)+ monkeypatch.setattr(+ m, "_mysql_replication_connection", lambda *_a, **_k: FenceConnection(),+ )+ monkeypatch.setattr(m, "_set_mysql_writer_fence", lambda *_a, **_k: True)+ monkeypatch.setattr(+ m, "_wait_for_mysql_writer_quiescence", lambda *_a, **_k: "source:1-9",+ )+ monkeypatch.setattr(m, "_wait_for_mysql_gtid", lambda *_a, **_k: True)+ monkeypatch.setattr(+ m,+ "_reset_mysql_replication_channel",+ lambda _cluster_id, cluster, **_kwargs: events.append(("reset", cluster))+ or True,+ )+ monkeypatch.setattr(+ m,+ "_configure_mysql_replica_from_source",+ lambda _cluster_id, replica, source, **_kwargs: events.append(+ ("configure", replica, source),+ ) or True,+ )+ monkeypatch.setattr(+ m,+ "_set_mysql_cluster_writable",+ lambda cluster, **_kwargs: events.append(("writable", cluster)) or True,+ )++ def fail_sync(*_args):+ raise RuntimeError("credential sync failed")++ monkeypatch.setattr(m, "_sync_global_mysql_credentials", fail_sync)++ result = m._planned_mysql_global_writer_switch(_mysql_switchover_params())++ assert result[0] == 400+ assert writer_member["IsWriter"] is False+ assert target_member["IsWriter"] is True+ assert events == [("reset", target), ("configure", writer, target)]+ owner = m._active_mysql_global_writer_switch(global_cluster)+ assert owner is not None and owner["state"] == "repair_required"+ assert global_cluster[m._MYSQL_GLOBAL_SWITCHOVER_STATE]["state"] == (+ "repair_required"+ )+ assert global_cluster["Status"] == "available"+ assert b"<Status>switching-over</Status>" in m._describe_global_clusters({+ "GlobalClusterIdentifier": "global-repl",+ })[2]+++ "rollback_failure",+ [+ "source-reset", "target-restore", "target-health", "target-fence",+ "source-writable", "final-persist",+ ],+)+def test_rds_switchover_failed_compensation_retains_owner(+ monkeypatch,+ rollback_failure,+):+ from ministack.services import rds as m++ writer, target, writer_member, target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )++ class FenceConnection:+ def close(self):+ pass++ monkeypatch.setattr(m, "_ensure_mysql_control_user", lambda *_a, **_k: True)+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)+ monkeypatch.setattr(m, "_mysql_writer_fence_active", lambda *_a, **_k: True)+ monkeypatch.setattr(m, "_mysql_replication_connection", lambda *_a, **_k: FenceConnection())+ monkeypatch.setattr(m, "_set_mysql_writer_fence", lambda _conn, **_kwargs: True)+ monkeypatch.setattr(m, "_wait_for_mysql_writer_quiescence", lambda *_a, **_k: "source:1-9")+ monkeypatch.setattr(m, "_wait_for_mysql_gtid", lambda *_a, **_k: True)+ def reset(_cluster_id, cluster, **_kwargs):+ if rollback_failure == "source-reset" and cluster is writer:+ raise RuntimeError("source reset failed")+ return True++ monkeypatch.setattr(m, "_reset_mysql_replication_channel", reset)++ configured = []+ def configure(_cluster_id, replica, _source, **_kwargs):+ configured.append(replica)+ if replica is writer:+ return False+ return rollback_failure != "target-restore"++ monkeypatch.setattr(m, "_configure_mysql_replica_from_source", configure)+ monkeypatch.setattr(+ m,+ "_wait_for_mysql_replication_healthy",+ lambda replica, _source, **_kwargs: not (+ rollback_failure == "target-health" and replica is target+ ),+ )+ writable = []+ monkeypatch.setattr(+ m, "_set_mysql_cluster_writable",+ lambda cluster, **_kwargs: writable.append(cluster)+ or rollback_failure != "source-writable",+ )+ target_safety = []+ if rollback_failure in ("target-fence", "final-persist"):+ monkeypatch.setattr(+ m,+ "_fence_or_contain_mysql_cluster",+ lambda cluster: target_safety.append(cluster) or (+ "contained" if rollback_failure == "target-fence" else "fenced"+ ),+ )+ monkeypatch.setattr(+ m,+ "_durably_persist_mysql_global_writer_switch_state",+ lambda _gc, marker: not (+ rollback_failure == "final-persist" and marker is None+ ),+ )++ result = m._planned_mysql_global_writer_switch(_mysql_switchover_params())++ assert result[0] == 400+ assert writer_member["IsWriter"] is True+ assert target_member["IsWriter"] is False+ assert target in configured+ assert writable == (+ []+ if rollback_failure in (+ "source-reset", "target-restore", "target-health", "target-fence",+ )+ else [writer]+ )+ assert target_safety == (+ [target]+ if rollback_failure == "target-fence"+ else [target, writer, target]+ if rollback_failure == "final-persist"+ else []+ )+ owner = m._active_mysql_global_writer_switch(global_cluster)+ assert owner is not None and owner["state"] == "repair_required"+ marker = global_cluster[m._MYSQL_GLOBAL_SWITCHOVER_STATE]+ assert marker["state"] == "repair_required"+ assert marker["source_arn"] == writer["DBClusterArn"]+ assert marker["target_arn"] == target["DBClusterArn"]+ m._mysql_global_writer_switch_owners.clear()+ assert m._active_mysql_global_writer_switch(global_cluster) is marker+ assert m._mysql_global_writer_switch_mutation_error(writer)[0] == 400+ assert global_cluster["Status"] == "available"+ assert b"<Status>switching-over</Status>" in m._describe_global_clusters({+ "GlobalClusterIdentifier": "global-repl",+ })[2]+++def test_rds_switchover_deadline_starts_before_preflight(monkeypatch):+ from ministack.services import rds as m++ _mysql_switchover_unit_topology(monkeypatch, m)+ now = [100.0]+ deadlines = []+ monkeypatch.setattr(m.time, "monotonic", lambda: now[0])++ def ensure_control(*_args, **kwargs):+ deadlines.append(kwargs["deadline"])+ now[0] += 2+ return len(deadlines) == 1++ monkeypatch.setattr(m, "_ensure_mysql_control_user", ensure_control)++ result = m._planned_mysql_global_writer_switch(_mysql_switchover_params())++ assert result[0] == 400+ assert deadlines == [130.0, 130.0]+++def test_rds_switchover_initial_lock_respects_deadline(monkeypatch):+ from ministack.services import rds as m++ _mysql_switchover_unit_topology(monkeypatch, m)+ acquired = threading.Event()+ release = threading.Event()++ def hold_lifecycle_lock():+ with m._shared_container_lock:+ acquired.set()+ release.wait(timeout=1)++ holder = threading.Thread(target=hold_lifecycle_lock)+ holder.start()+ assert acquired.wait(timeout=1)+ monkeypatch.setattr(m, "_MYSQL_GLOBAL_SWITCHOVER_TIMEOUT", 0.02)+ monkeypatch.setattr(+ m,+ "_resolve_global_writer_switch",+ lambda *_a, **_k: pytest.fail("timed-out request resolved topology"),+ )++ try:+ started = time.monotonic()+ result = m._planned_mysql_global_writer_switch(+ _mysql_switchover_params(),+ )+ elapsed = time.monotonic() - started+ finally:+ release.set()+ holder.join(timeout=1)++ assert result[0] == 400+ assert elapsed < 0.5+ assert not holder.is_alive()+++def test_rds_switchover_lock_deadline_never_claims_owner(monkeypatch):+ from ministack.services import rds as m++ writer, target, writer_member, target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )+ acquired = threading.Event()+ release = threading.Event()+ holder = None++ def hold_lifecycle_lock():+ with m._shared_container_lock:+ acquired.set()+ release.wait(timeout=1)++ def ensure_replication_user(*_args, **_kwargs):+ nonlocal holder+ holder = threading.Thread(target=hold_lifecycle_lock)+ holder.start()+ assert acquired.wait(timeout=1)+ return True++ monkeypatch.setattr(m, "_MYSQL_GLOBAL_SWITCHOVER_TIMEOUT", 0.02)+ monkeypatch.setattr(m, "_ensure_mysql_control_user", lambda *_a, **_k: True)+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", ensure_replication_user)+ monkeypatch.setattr(m, "_mysql_writer_fence_active", lambda *_a, **_k: True)++ try:+ result = m._planned_mysql_global_writer_switch(+ _mysql_switchover_params(),+ )+ finally:+ release.set()+ if holder:+ holder.join(timeout=1)++ assert result[0] == 400+ assert m._active_mysql_global_writer_switch(global_cluster) is None+ assert writer_member["IsWriter"] is True+ assert target_member["IsWriter"] is False+ assert writer.get("_mysql_replication_source_arn") is None+ assert target["_mysql_replication_source_arn"] == writer["DBClusterArn"]+++def test_rds_mysql_gtid_wait_recomputes_shared_deadline(monkeypatch):+ from ministack.services import rds as m++ now = [100.0]+ observed = {}+ monkeypatch.setattr(m.time, "monotonic", lambda: now[0])++ class Cursor:+ def __init__(self, connection):+ self.connection = connection++ def execute(self, statement, params):+ observed["sql"] = (statement, params)++ def fetchone(self):+ return (0,)++ def close(self):+ pass++ class Connection:+ def cursor(self):+ now[0] += 4+ return Cursor(self)++ def close(self):+ pass++ def connect(_cluster, *, timeout):+ observed["connect_timeout"] = timeout+ return Connection()++ monkeypatch.setattr(m, "_mysql_replication_connection", connect)++ assert m._wait_for_mysql_gtid(+ {"DBClusterIdentifier": "secondary"}, "source:1-9", 30,+ deadline=130.0,+ ) is True+ assert observed["connect_timeout"] == 30.0+ assert observed["sql"][1] == ("source:1-9", 26.0)+++def test_rds_mysql_replication_health_waits_for_expected_running_source(monkeypatch):+ from ministack.services import rds as m++ rows = iter([+ ("Connecting", "Yes", "172.20.0.20", 3306),+ ("Yes", "Yes", "172.20.0.20", 3306),+ ])+ statements = []+ sleeps = []++ class Cursor:+ description = [+ ("Replica_IO_Running",),+ ("Replica_SQL_Running",),+ ("Source_Host",),+ ("Source_Port",),+ ]++ def __init__(self, connection):+ self.connection = connection++ def execute(self, statement):+ statements.append(statement)++ def fetchone(self):+ return next(rows)++ def close(self):+ pass++ class Connection:+ def cursor(self):+ return Cursor(self)++ def close(self):+ pass++ monkeypatch.setattr(+ m, "_mysql_replication_connection", lambda *_a, **_k: Connection(),+ )+ monkeypatch.setattr(m.time, "sleep", sleeps.append)+ monkeypatch.setattr(m.time, "monotonic", lambda: 100.0)++ assert m._wait_for_mysql_replication_healthy(+ {"DBClusterIdentifier": "primary"},+ {"_shared_internal_address": "172.20.0.20", "_shared_internal_port": 3306},+ deadline=130.0,+ ) is True+ assert statements == ["SHOW REPLICA STATUS"] * 2+ assert sleeps == [m._MYSQL_REPLICATION_RETRY_INTERVAL]+++def test_rds_mysql_replication_health_rejects_wrong_source_at_deadline(monkeypatch):+ from ministack.services import rds as m++ now = iter([100.0, 100.0, 130.0])++ class Cursor:+ description = [+ ("Replica_IO_Running",),+ ("Replica_SQL_Running",),+ ("Source_Host",),+ ("Source_Port",),+ ]++ def __init__(self, connection):+ self.connection = connection++ def execute(self, _statement):+ pass++ def fetchone(self):+ return ("Yes", "Yes", "unexpected", 3306)++ def close(self):+ pass++ class Connection:+ def cursor(self):+ return Cursor(self)++ def close(self):+ pass++ monkeypatch.setattr(+ m, "_mysql_replication_connection", lambda *_a, **_k: Connection(),+ )+ monkeypatch.setattr(m.time, "monotonic", lambda: next(now))+ monkeypatch.setattr(+ m.time, "sleep", lambda *_a: pytest.fail("expired health check slept"),+ )++ assert m._wait_for_mysql_replication_healthy(+ {"DBClusterIdentifier": "primary"},+ {"_shared_internal_address": "172.20.0.20", "_shared_internal_port": 3306},+ deadline=130.0,+ ) is False++[email protected]("mutation", ["password", "topology", "failover"])+def test_rds_active_switchover_rejects_mutation(monkeypatch, mutation):+ from ministack.services import rds as m++ writer, target, writer_member, target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )+ with m._shared_container_lock:+ owner = m._claim_mysql_global_writer_switch(global_cluster, writer, target)+ if mutation == "password":+ result = m._modify_db_cluster({+ "DBClusterIdentifier": "primary",+ "MasterUserPassword": "changed-password",+ })+ elif mutation == "topology":+ result = m._remove_from_global_cluster({+ "GlobalClusterIdentifier": "global-repl",+ "DbClusterIdentifier": "secondary",+ })+ else:+ result = m._failover_global_cluster({+ "GlobalClusterIdentifier": "global-repl",+ "TargetDbClusterIdentifier": "secondary",+ "AllowDataLoss": "true",+ })++ assert result[0] == 400+ assert m._active_mysql_global_writer_switch(global_cluster) is owner+ assert writer_member["IsWriter"] is True+ assert target_member["IsWriter"] is False+ assert writer.get("_MasterUserPassword") != "changed-password"+ assert target["GlobalClusterIdentifier"] == "global-repl"+ assert target["_mysql_replication_source_arn"] == writer["DBClusterArn"]+++ ("handler_name", "initial_status"),+ [+ ("_start_db_instance", "stopped"),+ ("_stop_db_instance", "available"),+ ("_reboot_db_instance", "stopped"),+ ],+)+def test_rds_active_switchover_rejects_member_status_mutation(+ monkeypatch,+ handler_name,+ initial_status,+):+ from ministack.services import rds as m++ writer, target, _writer_member, _target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )+ instance = {+ "DBInstanceIdentifier": "instance-1",+ "DBClusterIdentifier": "primary",+ "DBInstanceStatus": initial_status,+ }+ monkeypatch.setattr(m, "_resolve_instance", lambda _identifier: instance)+ with m._shared_container_lock:+ owner = m._claim_mysql_global_writer_switch(global_cluster, writer, target)++ result = getattr(m, handler_name)({"DBInstanceIdentifier": "instance-1"})++ assert result[0] == 400+ assert instance["DBInstanceStatus"] == initial_status+ assert m._active_mysql_global_writer_switch(global_cluster) is owner+++ ("handler_name", "impl_name"),+ [+ ("_create_db_instance", "_create_db_instance_impl"),+ ("_delete_db_instance", "_delete_db_instance_impl"),+ ],+)+def test_rds_instance_mutation_prevents_switchover_claim(+ monkeypatch,+ handler_name,+ impl_name,+):+ from ministack.services import rds as m++ writer, target, _writer_member, _target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )+ entered = threading.Event()+ release = threading.Event()+ claimed = threading.Event()+ order = []++ def mutation_impl(_params):entered.set()assert release.wait(timeout=1)order.append("mutation")@@ -10998,6 +12177,139 @@ def test_aurora_mysql_control_user_can_inventory_writer_transactions():_delete_global_cluster(east, global_id)+ not os.environ.get("DOCKER_NETWORK"),+ reason="DOCKER_NETWORK not set -- live Aurora global switchover",+)+def test_aurora_mysql_global_switchover_relinks_data_plane():+ import pymysql++ east = _regional_rds("us-east-1")+ west = _regional_rds("us-west-2")+ suffix = uuid.uuid4().hex[:10]+ global_id = f"global-switch-live-{suffix}"+ primary_id = f"global-switch-live-primary-{suffix}"+ primary_instance_id = f"{primary_id}-writer"+ secondary_id = f"global-switch-live-secondary-{suffix}"+ secondary_instance_id = f"{secondary_id}-reader"+ primary_arn = None+ secondary_arn = None++ try:+ primary = east.create_db_cluster(+ DBClusterIdentifier=primary_id,+ Engine="aurora-mysql",+ EngineVersion="8.0.mysql_aurora.3.10.3",+ MasterUsername="admin",+ MasterUserPassword=PASSWORD,+ DatabaseName=DATABASE,+ )["DBCluster"]+ primary_arn = primary["DBClusterArn"]+ east.create_db_instance(+ DBInstanceIdentifier=primary_instance_id,+ DBClusterIdentifier=primary_id,+ DBInstanceClass="db.r6g.large",+ Engine="aurora-mysql",+ )+ primary_instance = _wait_for_instance(east, primary_instance_id)+ east.create_global_cluster(+ GlobalClusterIdentifier=global_id,+ SourceDBClusterIdentifier=primary_arn,+ )+ secondary = west.create_db_cluster(+ DBClusterIdentifier=secondary_id,+ Engine="aurora-mysql",+ EngineVersion="8.0.mysql_aurora.3.10.3",+ GlobalClusterIdentifier=global_id,+ MasterUsername="admin",+ MasterUserPassword=PASSWORD,+ DatabaseName=DATABASE,+ )["DBCluster"]+ secondary_arn = secondary["DBClusterArn"]+ west.create_db_instance(+ DBInstanceIdentifier=secondary_instance_id,+ DBClusterIdentifier=secondary_id,+ DBInstanceClass="db.r6g.large",+ Engine="aurora-mysql",+ )+ secondary_instance = _wait_for_instance(west, secondary_instance_id)+ _wait_for_replica_status(secondary_instance["Endpoint"])++ with _aurora_connect(primary_instance["Endpoint"]) as conn:+ with conn.cursor() as cursor:+ cursor.execute(+ "CREATE TABLE switchover_rows "+ "(id INT PRIMARY KEY, value VARCHAR(32))"+ )+ cursor.execute(+ "INSERT INTO switchover_rows VALUES (1, 'old-writer')"+ )+ _wait_for_gtid(+ primary_instance["Endpoint"], secondary_instance["Endpoint"],+ )++ response = east.switchover_global_cluster(+ GlobalClusterIdentifier=global_id,+ TargetDbClusterIdentifier=secondary_arn,+ )["GlobalCluster"]+ assert response["Status"] == "switching-over"+ assert response["FailoverState"] == {+ "Status": "pending",+ "FromDbClusterArn": primary_arn,+ "ToDbClusterArn": secondary_arn,+ "IsDataLossAllowed": False,+ }+ members = {+ member["DBClusterArn"]: member+ for member in response["GlobalClusterMembers"]+ }+ assert members[primary_arn]["IsWriter"] is False+ assert members[secondary_arn]["IsWriter"] is True+ final = west.describe_global_clusters(+ GlobalClusterIdentifier=global_id,+ )["GlobalClusters"][0]+ final_members = {+ member["DBClusterArn"]: member+ for member in final["GlobalClusterMembers"]+ }+ assert final["Status"] == "available"+ assert "FailoverState" not in final+ assert final_members[primary_arn]["IsWriter"] is False+ assert final_members[secondary_arn]["IsWriter"] is True+ assert final_members[secondary_arn]["Readers"] == [primary_arn]++ with _aurora_connect(secondary_instance["Endpoint"]) as conn:+ with conn.cursor() as cursor:+ cursor.execute("SELECT id, value FROM switchover_rows")+ assert cursor.fetchall() == ((1, "old-writer"),)+ cursor.execute(+ "INSERT INTO switchover_rows VALUES (2, 'new-writer')"+ )+ _wait_for_gtid(+ secondary_instance["Endpoint"], primary_instance["Endpoint"],+ )+ with _aurora_connect(primary_instance["Endpoint"]) as conn:+ with conn.cursor() as cursor:+ cursor.execute(+ "SELECT id, value FROM switchover_rows ORDER BY id"+ )+ assert cursor.fetchall() == (+ (1, "old-writer"),+ (2, "new-writer"),+ )+ with pytest.raises(pymysql.err.OperationalError) as exc:+ cursor.execute(+ "INSERT INTO switchover_rows VALUES (3, 'old-write')"+ )+ assert exc.value.args[0] == 1290+ finally:+ _cleanup_two_member_global(east, global_id, primary_arn, secondary_arn)+ _delete_instance(west, secondary_instance_id)+ _delete_instance(east, primary_instance_id)+ _delete_cluster(west, secondary_id)+ _delete_cluster(east, primary_id)++@pytest.mark.skipif(not os.environ.get("DOCKER_NETWORK"),reason="DOCKER_NETWORK not set -- live Aurora global replication",