Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
feat(rds): add Aurora MySQL promotion foundations (#1606)
Ian Oberstd6bcf2b8b921
diff --git a/ministack/services/rds.py b/ministack/services/rds.pyindex 324738e4..c10a339d 100644--- a/ministack/services/rds.py+++ b/ministack/services/rds.py@@ -120,6 +120,9 @@ _port_counter = [BASE_PORT]_docker = None_ministack_network = None_shared_container_lock = threading.RLock()+_mysql_global_writer_switch_lock = threading.RLock()+_mysql_global_writer_switch_owners = {}+_mysql_global_instance_mutations = {}_MYSQL_REPLICATION_USER = "rdsrepladmin"_MYSQL_REPLICATION_PASSWORD = "ministack-rds-replication"@@ -2407,11 +2410,11 @@ def _mysql_cluster_connection(cluster, user, password, *, timeout=None):)-def _mysql_admin_connection(cluster):- return _mysql_cluster_connection(- cluster,- "root",- cluster.get("_MasterUserPassword", "password"),+def _mysql_admin_connection(cluster, *, timeout=None):+ args = (cluster, "root", cluster.get("_MasterUserPassword", "password"))+ return (+ _mysql_cluster_connection(*args)+ if timeout is None else _mysql_cluster_connection(*args, timeout=timeout))@@ -2505,9 +2508,41 @@ def _ensure_mysql_compatibility(def _mysql_replication_connection(cluster, *, timeout=None):args = (cluster, _MYSQL_CONTROL_USER, _MYSQL_CONTROL_PASSWORD)- if timeout is None:- return _mysql_cluster_connection(*args)- return _mysql_cluster_connection(*args, timeout=timeout)+ return (+ _mysql_cluster_connection(*args)+ if timeout is None else _mysql_cluster_connection(*args, timeout=timeout)+ )+++def _mysql_deadline_timeout(*, timeout=None, deadline=None):+ if deadline is None:+ return timeout+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ raise TimeoutError("MySQL operation deadline elapsed")+ return remaining+++def _connect_mysql(connect, cluster, timeout):+ return connect(cluster) if timeout is None else connect(cluster, timeout=timeout)+++def _execute_mysql(cur, statement, params=None, *, deadline=None):+ if deadline is not None:+ timeout = _mysql_deadline_timeout(deadline=deadline)+ cur.connection._read_timeout = timeout+ cur.connection._write_timeout = timeout+ return cur.execute(statement) if params is None else cur.execute(statement, params)+++def _acquire_mysql_lifecycle_lock(deadline):+ remaining = deadline - time.monotonic()+ if remaining <= 0 or not _shared_container_lock.acquire(timeout=remaining):+ return False+ if time.monotonic() <= deadline:+ return True+ _shared_container_lock.release()+ return Falsedef _close_mysql_resources(operation, **resources):@@ -2526,11 +2561,17 @@ def _close_mysql_resources(operation, **resources):return succeeded-def _ensure_mysql_control_user(cluster, *, require_quiescence_grants=False):+def _ensure_mysql_control_user(+ cluster, *, require_quiescence_grants=False, timeout=None, deadline=None,+):"""Create a local, non-replicated account for replica lifecycle SQL."""- if cluster.get("_mysql_control_user_ready") and (+ if (+ cluster.get("_mysql_control_user_ready")+ and cluster.get("_mysql_replication_status_grant_ready")+ and (not require_quiescence_grantsor cluster.get("_mysql_writer_quiescence_grants_ready")+ )):return Truecluster_id = cluster.get("DBClusterIdentifier", "unknown")@@ -2538,26 +2579,35 @@ def _ensure_mysql_control_user(cluster, *, require_quiescence_grants=False):cur = Nonesucceeded = Falsetry:- conn = _mysql_admin_connection(cluster)+ conn = _connect_mysql(+ _mysql_admin_connection, cluster,+ _mysql_deadline_timeout(timeout=timeout, deadline=deadline),+ )if conn is None:return Falsecur = conn.cursor()- cur.execute("SET SESSION sql_log_bin=0")- cur.execute(+ _execute_mysql(cur, "SET SESSION sql_log_bin=0", deadline=deadline)+ _execute_mysql(+ cur,"CREATE USER IF NOT EXISTS %s@'%%' ""IDENTIFIED WITH mysql_native_password BY %s",(_MYSQL_CONTROL_USER, _MYSQL_CONTROL_PASSWORD),+ deadline=deadline,)- cur.execute(- "GRANT PROCESS, RELOAD ON *.* TO %s@'%%'",+ _execute_mysql(+ cur,+ "GRANT PROCESS, RELOAD, REPLICATION CLIENT ON *.* TO %s@'%%'",(_MYSQL_CONTROL_USER,),+ deadline=deadline,)- cur.execute(+ _execute_mysql(+ cur,"GRANT CONNECTION_ADMIN, REPLICATION_SLAVE_ADMIN, ""SYSTEM_VARIABLES_ADMIN, XA_RECOVER_ADMIN ON *.* TO %s@'%%'",(_MYSQL_CONTROL_USER,),+ deadline=deadline,)- cur.execute("FLUSH PRIVILEGES")+ _execute_mysql(cur, "FLUSH PRIVILEGES", deadline=deadline)succeeded = Trueexcept Exception as e:logger.warning(@@ -2570,17 +2620,20 @@ def _ensure_mysql_control_user(cluster, *, require_quiescence_grants=False):succeeded = Falseif succeeded:cluster["_mysql_control_user_ready"] = True+ cluster["_mysql_replication_status_grant_ready"] = Truecluster["_mysql_writer_quiescence_grants_ready"] = Truereturn succeeded-def _set_mysql_writer_fence(conn):+def _set_mysql_writer_fence(conn, *, deadline=None):"""Fence on a caller-owned connection; never close or bound that owner."""cur = Nonesucceeded = Falsetry:cur = conn.cursor()- cur.execute("SET GLOBAL super_read_only=ON")+ _execute_mysql(+ cur, "SET GLOBAL super_read_only=ON", deadline=deadline,+ )succeeded = Trueexcept Exception as e:logger.warning("RDS: failed to install MySQL writer fence: %s", e)@@ -2617,15 +2670,11 @@ def _mysql_control_query(cluster, statement, *, deadline=None):timeout = deadline - time.monotonic()if timeout <= 0:return None- conn = (- _mysql_replication_connection(cluster)- if timeout is None- else _mysql_replication_connection(cluster, timeout=timeout)- )+ conn = _connect_mysql(_mysql_replication_connection, cluster, timeout)if conn is None:return Nonecur = conn.cursor()- cur.execute(statement)+ _execute_mysql(cur, statement, deadline=deadline)result = cur.fetchall()if isinstance(result, (tuple, list)) and (deadline is None or time.monotonic() <= deadline@@ -2712,7 +2761,9 @@ def _capture_mysql_gtid_executed(cluster, *, deadline=None):return gtid if isinstance(gtid, str) and gtid and gtid == gtid.strip() else None-def _wait_for_mysql_writer_quiescence(cluster, timeout, poll_interval=0.1):+def _wait_for_mysql_writer_quiescence(+ cluster, timeout, poll_interval=0.1, *, deadline=None,+):"""Naturally drain a separately fenced writer and capture its GTID set."""if (isinstance(timeout, bool)@@ -2733,7 +2784,14 @@ def _wait_for_mysql_writer_quiescence(cluster, timeout, poll_interval=0.1):or poll_interval <= 0):return None- deadline = time.monotonic() + timeout+ if deadline is None:+ deadline = time.monotonic() + timeout+ elif (+ isinstance(deadline, bool)+ or not isinstance(deadline, (int, float))+ or not math.isfinite(deadline)+ ):+ return Nonecontainer_epoch = cluster.get("_shared_container_epoch")def _retry():@@ -2795,42 +2853,149 @@ def _wait_for_mysql_writer_quiescence(cluster, timeout, poll_interval=0.1):return gtid-def _ensure_mysql_replication_user(cluster):+def _wait_for_mysql_gtid(cluster, gtid, timeout, *, deadline=None):+ if (+ not isinstance(gtid, str)+ or not gtid+ or gtid != gtid.strip()+ or isinstance(timeout, bool)+ or not isinstance(timeout, (int, float))+ ):+ return False+ try:+ timeout = float(timeout)+ except (OverflowError, ValueError):+ return False+ if not math.isfinite(timeout) or timeout <= 0:+ return False++ conn = None+ cur = None+ succeeded = False+ try:+ conn = _connect_mysql(+ _mysql_replication_connection, cluster,+ _mysql_deadline_timeout(timeout=timeout, deadline=deadline),+ )+ if conn is None:+ return False+ cur = conn.cursor()+ wait_timeout = _mysql_deadline_timeout(timeout=timeout, deadline=deadline)+ _execute_mysql(+ cur,+ "SELECT WAIT_FOR_EXECUTED_GTID_SET(%s, %s)",+ (gtid, wait_timeout),+ deadline=deadline,+ )+ row = cur.fetchone()+ succeeded = bool(+ isinstance(row, (tuple, list))+ and len(row) == 1+ and type(row[0]) is int+ and row[0] == 0+ )+ except Exception as e:+ logger.warning(+ "RDS: failed waiting for MySQL GTID convergence on %s: %s",+ cluster.get("DBClusterIdentifier", "unknown"), e,+ )+ finally:+ if not _close_mysql_resources("GTID wait", cursor=cur, connection=conn):+ succeeded = False+ return succeeded+++def _set_mysql_cluster_writable(cluster, *, timeout=None, deadline=None):+ """Enable and verify writes, restoring the fence on partial failure."""+ conn = None+ cur = None+ super_read_only_disable_attempted = False+ succeeded = False+ try:+ conn = _connect_mysql(+ _mysql_replication_connection, cluster,+ _mysql_deadline_timeout(timeout=timeout, deadline=deadline),+ )+ if conn is None:+ return False+ cur = conn.cursor()+ super_read_only_disable_attempted = True+ _execute_mysql(cur, "SET GLOBAL super_read_only=OFF", deadline=deadline)+ _execute_mysql(cur, "SET GLOBAL read_only=OFF", deadline=deadline)+ _execute_mysql(+ cur, "SELECT @@GLOBAL.read_only, @@GLOBAL.super_read_only",+ deadline=deadline,+ )+ row = cur.fetchone()+ succeeded = bool(+ isinstance(row, (tuple, list))+ and len(row) == 2+ and all(type(value) is int for value in row)+ and tuple(row) == (0, 0)+ )+ except Exception as e:+ logger.warning(+ "RDS: failed enabling MySQL writes on %s: %s",+ cluster.get("DBClusterIdentifier", "unknown"), e,+ )+ finally:+ if not succeeded and super_read_only_disable_attempted and cur is not None:+ try:+ # Safety compensation uses the connection's last bounded timeout.+ cur.execute("SET GLOBAL super_read_only=ON")+ except Exception as e:+ logger.warning(+ "RDS: failed restoring MySQL fence on %s: %s",+ cluster.get("DBClusterIdentifier", "unknown"), e,+ )+ _close_mysql_resources("write enable", cursor=cur, connection=conn)+ return succeeded+++def _ensure_mysql_replication_user(cluster, *, timeout=None, deadline=None):"""Ensure the internal replication account exists on a writer cluster."""cluster_id = cluster.get("DBClusterIdentifier", "unknown")conn = Nonecur = None+ succeeded = Falsetry:- conn = _mysql_admin_connection(cluster)+ conn = _connect_mysql(+ _mysql_admin_connection, cluster,+ _mysql_deadline_timeout(timeout=timeout, deadline=deadline),+ )if conn is None:return Falsecur = conn.cursor()- cur.execute(+ _execute_mysql(+ cur,"CREATE USER IF NOT EXISTS %s@'%%' ""IDENTIFIED WITH mysql_native_password BY %s",(_MYSQL_REPLICATION_USER, _MYSQL_REPLICATION_PASSWORD),+ deadline=deadline,)- cur.execute(+ _execute_mysql(+ cur,"GRANT REPLICATION SLAVE ON *.* TO %s@'%%'",(_MYSQL_REPLICATION_USER,),+ deadline=deadline,)- cur.execute("FLUSH PRIVILEGES")- return True+ _execute_mysql(cur, "FLUSH PRIVILEGES", deadline=deadline)+ succeeded = Trueexcept Exception as e:logger.warning("RDS: failed to prepare MySQL replication source %s: %s",cluster_id,e,)- return Falsefinally:- if cur is not None:- cur.close()- if conn is not None:- conn.close()+ if not _close_mysql_resources("replication user", cursor=cur, connection=conn):+ succeeded = False+ return succeeded-def _configure_mysql_replica_from_source(cluster_id, replica, source):+def _configure_mysql_replica_from_source(+ cluster_id, replica, source, *, timeout=None, deadline=None,+):"""Configure a replica channel from an explicit source cluster.The caller owns role validation and must ensure the replication account is@@ -2842,13 +3007,17 @@ def _configure_mysql_replica_from_source(cluster_id, replica, source):"""conn = Nonecur = None+ succeeded = Falsetry:- conn = _mysql_replication_connection(replica)+ conn = _connect_mysql(+ _mysql_replication_connection, replica,+ _mysql_deadline_timeout(timeout=timeout, deadline=deadline),+ )if conn is None:- return False+ raise RuntimeError("MySQL replication connection is unavailable")cur = conn.cursor()try:- cur.execute("STOP REPLICA")+ _execute_mysql(cur, "STOP REPLICA", deadline=deadline)except Exception as e:logger.debug("RDS: STOP REPLICA was a no-op for %s: %s",@@ -2860,8 +3029,9 @@ def _configure_mysql_replica_from_source(cluster_id, replica, source):# and relay logs but deliberately preserves gtid_executed. Within# the source's configured retention window, auto-position can# request every transaction not yet applied before the restart.- cur.execute("RESET REPLICA ALL")- cur.execute(+ _execute_mysql(cur, "RESET REPLICA ALL", deadline=deadline)+ _execute_mysql(+ cur,"CHANGE REPLICATION SOURCE TO ""SOURCE_HOST=%s, SOURCE_PORT=%s, SOURCE_USER=%s, ""SOURCE_PASSWORD=%s, SOURCE_AUTO_POSITION=1, "@@ -2872,21 +3042,25 @@ def _configure_mysql_replica_from_source(cluster_id, replica, source):_MYSQL_REPLICATION_USER,_MYSQL_REPLICATION_PASSWORD,),+ deadline=deadline,)- cur.execute("START REPLICA")- cur.execute("SET GLOBAL super_read_only=ON")+ _execute_mysql(cur, "START REPLICA", deadline=deadline)+ _execute_mysql(cur, "SET GLOBAL super_read_only=ON", deadline=deadline)+ succeeded = Trueexcept Exception as e:logger.warning("RDS: failed to configure MySQL replication for %s: %s",cluster_id,e,)- return Falsefinally:- if cur is not None:- cur.close()- if conn is not None:- conn.close()+ if not _close_mysql_resources(+ "replication configuration", cursor=cur, connection=conn,+ ):+ succeeded = False++ if not succeeded:+ return Falsereplica["_mysql_replication_source_arn"] = source.get("DBClusterArn")replica.pop("_mysql_replication_reset_pending", None)@@ -2912,6 +3086,8 @@ def _configure_mysql_replication(cluster_id, cluster):"""if not _aurora_mysql_8_replication_enabled(cluster):return None+ if _mysql_global_writer_switch_mutation_error(cluster):+ return Falseglobal_cluster, member = _global_cluster_member_for_cluster(cluster)if not global_cluster or not member:@@ -2974,6 +3150,9 @@ def _reset_mysql_replication_channel(cluster,*,clear_super_read_only=True,+ allow_missing_channel=False,+ timeout=None,+ deadline=None,):"""Stop and reset a replica channel without consulting global role metadata.@@ -2983,12 +3162,18 @@ def _reset_mysql_replication_channel(source/detach metadata for its caller to clear and, when requested, clearsonly ``super_read_only`` to preserve existing detach behavior. It does notguarantee general-client writability while ``read_only`` remains enabled.+ Callers may explicitly tolerate an already-absent channel; that case is+ verified before destructive channel SQL and recorded as reset."""if not cluster.get("_shared_storage_initialized"):return Trueif not cluster.get("_shared_container_ready"):return False- if not _ensure_mysql_control_user(cluster):+ control_ready = (+ _ensure_mysql_control_user(cluster)+ if deadline is None else _ensure_mysql_control_user(cluster, deadline=deadline)+ )+ if not control_ready:return False# Persist the transition before destructive SQL. A failed request keeps@@ -3000,23 +3185,34 @@ def _reset_mysql_replication_channel(conn = Nonecur = Nonetry:- conn = _mysql_replication_connection(cluster)+ conn = _connect_mysql(+ _mysql_replication_connection, cluster,+ _mysql_deadline_timeout(timeout=timeout, deadline=deadline),+ )if conn is None:return Falsecur = conn.cursor()- if state == "requested":- cur.execute("STOP REPLICA")- cluster["_mysql_replication_detach_state"] = "stopped"- state = "stopped"- if state in ("stopped", "resetting"):- # ``resetting`` is written first so a crash between MySQL applying- # RESET and Python publishing the next state is recoverable.- cluster["_mysql_replication_detach_state"] = "resetting"- cur.execute("RESET REPLICA ALL")- cluster["_mysql_replication_detach_state"] = "reset"- state = "reset"+ if allow_missing_channel:+ _execute_mysql(cur, "SHOW REPLICA STATUS", deadline=deadline)+ if cur.fetchone() is None:+ cluster["_mysql_replication_detach_state"] = "reset"+ state = "reset"+ if state != "reset":+ if state == "requested":+ _execute_mysql(cur, "STOP REPLICA", deadline=deadline)+ cluster["_mysql_replication_detach_state"] = "stopped"+ state = "stopped"+ if state in ("stopped", "resetting"):+ # ``resetting`` is written first so a crash between MySQL applying+ # RESET and Python publishing the next state is recoverable.+ cluster["_mysql_replication_detach_state"] = "resetting"+ _execute_mysql(cur, "RESET REPLICA ALL", deadline=deadline)+ cluster["_mysql_replication_detach_state"] = "reset"+ state = "reset"if state == "reset" and clear_super_read_only:- cur.execute("SET GLOBAL super_read_only=OFF")+ _execute_mysql(+ cur, "SET GLOBAL super_read_only=OFF", deadline=deadline,+ )except Exception as e:logger.warning("RDS: failed to reset MySQL replication channel for %s: %s",@@ -3025,10 +3221,10 @@ def _reset_mysql_replication_channel()raisefinally:- if cur is not None:- cur.close()- if conn is not None:- conn.close()+ if not _close_mysql_resources(+ "replication reset", cursor=cur, connection=conn,+ ):+ raise RuntimeError("MySQL replication reset cleanup failed")return True@@ -3846,7 +4042,16 @@ def _unregister_instance_from_clusters(db_id):# ---------------------------------------------------------------------------def _create_db_instance(p):- return _create_db_instance_impl(p)+ cluster = None+ cluster_id = _p(p, "DBClusterIdentifier")+ cluster = _resolve_cluster_in_request_region(cluster_id) if cluster_id else None+ if cluster and (switch_error := _begin_mysql_global_instance_mutation(cluster)):+ return switch_error+ try:+ return _create_db_instance_impl(p)+ finally:+ if cluster:+ _end_mysql_global_instance_mutation(cluster)def _create_db_instance_impl(p):@@ -4472,6 +4677,22 @@ def _create_db_instance_impl(p):def _delete_db_instance(p):+ cluster = None+ instance = _resolve_instance(_p(p, "DBInstanceIdentifier"))+ cluster_id = instance and (+ instance.get("_shared_cluster_id") or instance.get("DBClusterIdentifier")+ )+ cluster = _resolve_cluster_in_request_region(cluster_id) if cluster_id else None+ if cluster and (switch_error := _begin_mysql_global_instance_mutation(cluster)):+ return switch_error+ try:+ return _delete_db_instance_impl(p)+ finally:+ if cluster:+ _end_mysql_global_instance_mutation(cluster)+++def _delete_db_instance_impl(p):db_id = _p(p, "DBInstanceIdentifier")instance = _resolve_instance(db_id)if not instance:@@ -4917,6 +5138,11 @@ def _serverlessv2_from_params(p):def _create_db_cluster(p):+ with _shared_container_lock:+ return _create_db_cluster_impl(p)+++def _create_db_cluster_impl(p):cluster_id = _p(p, "DBClusterIdentifier")if not cluster_id:return _error("MissingParameter", "DBClusterIdentifier is required", 400)@@ -4932,6 +5158,11 @@ def _create_db_cluster(p):if global_cluster_id and not global_cluster:return _error("GlobalClusterNotFoundFault",f"Global cluster {global_cluster_id} not found.", 404)+ if global_cluster:+ if switch_error := _mysql_global_writer_switch_mutation_error(+ _global_cluster_writer_cluster(global_cluster),+ ):+ return switch_errorif global_cluster and _global_cluster_member_in_request_region(global_cluster):return _error("InvalidParameterValue",@@ -5393,6 +5624,11 @@ def _apply_cluster_master_password_change(cluster, new_pass):def _modify_db_cluster(p):+ with _shared_container_lock:+ return _modify_db_cluster_impl(p)+++def _modify_db_cluster_impl(p):cluster_id = _p(p, "DBClusterIdentifier")cluster = _resolve_cluster_in_request_region(cluster_id)if not cluster:@@ -5400,6 +5636,8 @@ def _modify_db_cluster(p):if wrong_region:return wrong_regionreturn _error("DBClusterNotFoundFault", f"DBCluster {cluster_id} not found.", 404)+ if switch_error := _mysql_global_writer_switch_mutation_error(cluster):+ return switch_error# Validate the password parameters together before mutating anything, so# a rejected combination cannot leave a half-applied password change.@@ -6104,10 +6342,18 @@ def _modify_subnet_group(p):def _cluster_member_instances(cluster):"""Resolve a cluster's member records to their instance dicts."""+ parsed = _parse_rds_arn(cluster.get("DBClusterArn", ""))+ scope = parsed[0] if parsed else None++ def resolve(instance_id):+ return (+ _instances.get_scoped(scope.account_id, scope.region, instance_id)+ if scope else _instances.get(instance_id)+ )return [instfor inst in (- _instances.get(member.get("DBInstanceIdentifier"))+ resolve(member.get("DBInstanceIdentifier"))for member in cluster.get("DBClusterMembers", []))if inst is not None@@ -6497,6 +6743,11 @@ def _stop_db_cluster(p):def _failover_db_cluster(p):+ with _shared_container_lock:+ return _failover_db_cluster_impl(p)+++def _failover_db_cluster_impl(p):"""Force an intra-cluster failover: promote a reader member to writer.Metadata-only for now, mirroring ``_failover_global_cluster``: the member@@ -6522,6 +6773,8 @@ def _failover_db_cluster(p):if wrong_region:return wrong_regionreturn _error("DBClusterNotFoundFault", f"DBCluster {cluster_id} not found.", 404)+ if switch_error := _mysql_global_writer_switch_mutation_error(cluster):+ return switch_errorcluster_id = cluster.get("DBClusterIdentifier", cluster_id)status = cluster.get("Status")if status != "available":@@ -6911,6 +7164,15 @@ def _describe_global_clusters(p):gcs = [gc]else:gcs = list(_global_clusters.values())+ with _shared_container_lock:+ snapshots = []+ for gc in gcs:+ switching = bool(_active_mysql_global_writer_switch(gc))+ snapshot = copy.deepcopy(gc)+ if switching:+ snapshot["Status"] = "switching-over"+ snapshots.append(snapshot)+ gcs = snapshotsmembers_xml = "".join(f"<GlobalCluster>{_global_cluster_xml(gc)}</GlobalCluster>" for gc in gcs@@ -6957,6 +7219,8 @@ def _remove_from_global_cluster(p):with _shared_container_lock:members = gc.get("GlobalClusterMembers", [])cluster = _resolve_cluster(db_cluster_id)+ if switch_error := _mysql_global_writer_switch_mutation_error(cluster):+ return switch_errordb_cluster_arn = cluster["DBClusterArn"] if cluster else db_cluster_idmember = next((m for m in members if m["DBClusterArn"] == db_cluster_arn),@@ -7005,6 +7269,11 @@ def _remove_from_global_cluster(p):def _modify_global_cluster(p):+ with _shared_container_lock:+ return _modify_global_cluster_impl(p)+++def _modify_global_cluster_impl(p):gc_id = _p(p, "GlobalClusterIdentifier")invalid_id = _invalid_global_cluster_identifier_error(gc_id)if invalid_id:@@ -7013,6 +7282,10 @@ def _modify_global_cluster(p):if not gc:return _error("GlobalClusterNotFoundFault",f"Global cluster {gc_id} not found.", 404)+ if switch_error := _mysql_global_writer_switch_mutation_error(+ _global_cluster_writer_cluster(gc),+ ):+ return switch_errorengine_version = _p(p, "EngineVersion")engine_version_error = _unsupported_aurora_engine_version_error(@@ -7063,6 +7336,151 @@ def _find_global_cluster_target_member(gc, target_cluster_id):return cluster, member+def _active_mysql_global_writer_switch(gc):+ owner = _mysql_global_writer_switch_owners.get(id(gc)) if gc else None+ if owner and owner.get("global_cluster") is gc:+ return owner+ return None+++def _mysql_global_writer_switch_mutation_error(cluster):+ if not cluster:+ return None+ with _shared_container_lock:+ return _mysql_global_writer_switch_mutation_error_unlocked(cluster)+++def _mysql_global_writer_switch_mutation_error_unlocked(cluster):+ gc, _member = _global_cluster_member_for_cluster(cluster)+ if not _active_mysql_global_writer_switch(gc):+ return None+ return _error(+ "InvalidDBClusterStateFault",+ f"DBCluster {cluster.get('DBClusterIdentifier', '')} cannot be "+ "modified while a global writer switchover is in progress.",+ 400,+ )+++def _begin_mysql_global_instance_mutation(cluster):+ with _mysql_global_writer_switch_lock:+ if error := _mysql_global_writer_switch_mutation_error_unlocked(cluster):+ return error+ entry = _mysql_global_instance_mutations.setdefault(+ id(cluster), {"cluster": cluster, "count": 0},+ )+ entry["count"] += 1+ return None+++def _end_mysql_global_instance_mutation(cluster):+ with _mysql_global_writer_switch_lock:+ entry = _mysql_global_instance_mutations.get(id(cluster))+ if not entry or entry["cluster"] is not cluster:+ return+ entry["count"] -= 1+ if not entry["count"]:+ _mysql_global_instance_mutations.pop(id(cluster), None)+++def _active_mysql_global_instance_mutation(cluster):+ entry = _mysql_global_instance_mutations.get(id(cluster))+ return bool(entry and entry["cluster"] is cluster and entry["count"])+++def _mysql_global_writer_switch_topology(gc):+ return tuple(+ (member.get("DBClusterArn"), bool(member.get("IsWriter")))+ for member in gc.get("GlobalClusterMembers", [])+ )+++def _mysql_global_writer_switch_identity(cluster):+ compute = tuple(+ (instance.get("DBInstanceIdentifier"), instance.get("DBInstanceStatus"))+ for instance in _cluster_member_instances(cluster)+ )+ return (cluster.get("_shared_container_epoch"), cluster.get("_shared_container_id"), compute)+++def _mysql_global_writer_switch_ready(gc, source, target):+ return bool(+ gc.get("Status") == "available"+ and source.get("EngineVersion") == target.get("EngineVersion")+ and _mysql_gtid_history_ready(source)+ and _mysql_gtid_history_ready(target)+ and all(+ cluster.get("_shared_storage_initialized")+ and cluster.get("_shared_container_ready") is True+ and cluster.get("Status") == "available"+ and cluster.get("DBClusterMembers")+ and any(+ instance.get("DBInstanceStatus") == "available"+ for instance in _cluster_member_instances(cluster)+ )+ and cluster.get("_shared_container_id")+ and cluster.get("_shared_internal_address")+ and cluster.get("_shared_internal_port")+ and isinstance(cluster.get("_shared_container_epoch"), int)+ and cluster.get("_shared_container_epoch") > 0+ and not cluster.get("_pending_master_password_rotation")+ and not cluster.get("_mysql_replication_detach_state")+ and not _active_mysql_global_instance_mutation(cluster)+ for cluster in (source, target)+ )+ and not source.get("_mysql_replication_source_arn")+ and target.get("_mysql_replication_source_arn") == source.get("DBClusterArn")+ )+++def _claim_mysql_global_writer_switch(gc, source, target):+ with _mysql_global_writer_switch_lock:+ if _active_mysql_global_writer_switch(gc) or any(+ _active_mysql_global_instance_mutation(cluster)+ for cluster in (source, target)+ ):+ return None+ owner = {+ "global_cluster": gc,+ "source": source,+ "target": target,+ "topology": _mysql_global_writer_switch_topology(gc),+ "source_identity": _mysql_global_writer_switch_identity(source),+ "target_identity": _mysql_global_writer_switch_identity(target),+ }+ _mysql_global_writer_switch_owners[id(gc)] = owner+ return owner+++def _mysql_global_writer_switch_still_owned(owner, expected_topology):+ gc = owner["global_cluster"]+ source = owner["source"]+ target = owner["target"]+ return bool(+ _active_mysql_global_writer_switch(gc) is owner+ and _mysql_global_writer_switch_topology(gc) == expected_topology+ and _mysql_global_writer_switch_identity(source) == owner["source_identity"]+ and _mysql_global_writer_switch_identity(target) == owner["target_identity"]+ and source.get("_shared_container_ready") is True+ and target.get("_shared_container_ready") is True+ and source.get("_shared_storage_initialized") is True+ and target.get("_shared_storage_initialized") is True+ and not source.get("_pending_master_password_rotation")+ and not target.get("_pending_master_password_rotation")+ )+++def _finish_mysql_global_writer_switch(owner, *, repair_required=False):+ gc = owner["global_cluster"]+ if _active_mysql_global_writer_switch(gc) is not owner:+ return+ if repair_required:+ owner["state"] = "repair_required"+ else:+ _mysql_global_writer_switch_owners.pop(id(gc), None)+ gc["Status"] = "available"++def _switch_global_cluster_writer(p, *, allow_data_loss=False):gc_id = _p(p, "GlobalClusterIdentifier")target_cluster_id = _p(p, "TargetDbClusterIdentifier")@@ -9338,6 +9756,8 @@ def reset():# restore worker can pass its membership check after reset has enumerated# resources, then create a container after the stores are cleared.with _shared_container_lock:+ _mysql_global_writer_switch_owners.clear()+ _mysql_global_instance_mutations.clear()docker_client = _get_docker()if docker_client:# Shared containers are cluster-owned. Reap them once from the clusterdiff --git a/tests/test_rds.py b/tests/test_rds.pyindex 8cfc7f42..88ac9917 100644--- a/tests/test_rds.py+++ b/tests/test_rds.py@@ -8773,7 +8773,7 @@ def test_rds_mysql_replication_restore_resets_once_then_is_idempotent(monkeypatcdef close(self):pass- monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda _cluster: True)+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)monkeypatch.setattr(m,"_mysql_replication_connection",@@ -8838,7 +8838,7 @@ def test_rds_mysql_replication_writer_sweep_closes_reverse_readiness_race(def close(self):pass- monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda _cluster: True)+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)monkeypatch.setattr(m,"_mysql_replication_connection",@@ -8897,7 +8897,7 @@ def test_rds_mysql_replication_failure_is_retryable_and_closes_connections(def close(self):closed.append("connection")- monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda _cluster: True)+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)monkeypatch.setattr(m, "_mysql_replication_connection", lambda _cluster: FakeConnection())monkeypatch.setattr(m,@@ -9496,6 +9496,63 @@ def test_rds_mysql_replication_channel_reset_is_role_independent_and_keeps_fenceassert closed == ["cursor", "connection"]+ ("clear_super_read_only", "expected_statements"),+ [+ (True, ["SHOW REPLICA STATUS", "SET GLOBAL super_read_only=OFF"]),+ (False, ["SHOW REPLICA STATUS"]),+ ],+)+def test_rds_mysql_replication_channel_reset_allows_missing_channel(+ monkeypatch,+ clear_super_read_only,+ expected_statements,+):+ from ministack.services import rds as m++ writer, _secondary, _writer_member, _secondary_member, _global = (+ _mysql_replication_unit_topology()+ )+ writer["_shared_storage_initialized"] = True+ statements = []+ closed = []++ class FakeCursor:+ def execute(self, statement, params=None):+ statements.append((statement, params))++ def fetchone(self):+ return None++ def close(self):+ closed.append("cursor")++ class FakeConnection:+ def cursor(self):+ return FakeCursor()++ def close(self):+ closed.append("connection")++ monkeypatch.setattr(m, "_ensure_mysql_control_user", lambda _cluster: True)+ monkeypatch.setattr(+ m,+ "_mysql_replication_connection",+ lambda cluster: FakeConnection() if cluster is writer else None,+ )++ assert m._reset_mysql_replication_channel(+ "primary",+ writer,+ clear_super_read_only=clear_super_read_only,+ allow_missing_channel=True,+ ) is True++ assert [statement for statement, _params in statements] == expected_statements+ assert writer["_mysql_replication_detach_state"] == "reset"+ assert closed == ["cursor", "connection"]++@pytest.mark.parametrize("failed_statement",[@@ -9668,7 +9725,7 @@ def test_rds_mysql_replication_detach_failure_rolls_channel_back_atomically(def close(self):pass- monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda _cluster: True)+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)monkeypatch.setattr(m, "_mysql_replication_connection", lambda _cluster: FakeConnection())assert m._detach_mysql_replication("secondary", secondary) is False@@ -9725,7 +9782,7 @@ def test_rds_mysql_replication_detach_partial_rollback_retries_from_requested(def close(self):pass- monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda _cluster: True)+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)monkeypatch.setattr(m, "_mysql_replication_connection", lambda _cluster: FakeConnection())assert m._detach_mysql_replication("secondary", secondary) is False@@ -10005,6 +10062,7 @@ def test_rds_mysql_control_user_is_local_and_used_for_replica_sql(monkeypatch):cluster = {"DBClusterIdentifier": "secondary","_mysql_control_user_ready": True,+ "_mysql_replication_status_grant_ready": True,}events = []connection_args = []@@ -10025,7 +10083,7 @@ def test_rds_mysql_control_user_is_local_and_used_for_replica_sql(monkeypatch):"CREATE USER IF NOT EXISTS %s@'%%' ""IDENTIFIED WITH mysql_native_password BY %s"),- "GRANT PROCESS, RELOAD ON *.* TO %s@'%%'",+ "GRANT PROCESS, RELOAD, REPLICATION CLIENT ON *.* TO %s@'%%'",("GRANT CONNECTION_ADMIN, REPLICATION_SLAVE_ADMIN, ""SYSTEM_VARIABLES_ADMIN, XA_RECOVER_ADMIN ON *.* TO %s@'%%'"@@ -10033,6 +10091,7 @@ def test_rds_mysql_control_user_is_local_and_used_for_replica_sql(monkeypatch):"FLUSH PRIVILEGES",]assert statements[2][1] == statements[3][1] == (m._MYSQL_CONTROL_USER,)+ assert cluster["_mysql_replication_status_grant_ready"] is Trueassert cluster["_mysql_writer_quiescence_grants_ready"] is Trueassert m._ensure_mysql_control_user(cluster) is Trueassert m._ensure_mysql_control_user(@@ -10040,6 +10099,11 @@ def test_rds_mysql_control_user_is_local_and_used_for_replica_sql(monkeypatch):) is Trueassert len(events) == grant_count+ cluster.pop("_mysql_replication_status_grant_ready")+ assert m._ensure_mysql_control_user(cluster) is True+ assert len(events) == grant_count * 2+ assert cluster["_mysql_replication_status_grant_ready"] is True+monkeypatch.setattr(m,"_mysql_cluster_connection",@@ -10078,7 +10142,9 @@ def _mysql_test_connection(events, rows=(), failure=None):class FakeConnection:def cursor(self):- return FakeCursor()+ cursor = FakeCursor()+ cursor.connection = self+ return cursordef close(self):events.append("connection.close")@@ -10132,6 +10198,16 @@ def test_rds_mysql_writer_fence_keeps_caller_connection(monkeypatch, failure, exassert "connection.close" not in events+def test_rds_mysql_writer_fence_recomputes_shared_deadline(monkeypatch):+ from ministack.services import rds as m++ conn = _mysql_test_connection([])+ monkeypatch.setattr(m.time, "monotonic", lambda: 104.0)++ assert m._set_mysql_writer_fence(conn, deadline=110.0) is True+ assert conn._read_timeout == conn._write_timeout == 6.0++@pytest.mark.parametrize(("readback", "expected"),[@@ -10182,13 +10258,17 @@ def test_rds_mysql_control_query_uses_remaining_deadline():from ministack.services import rds as m- now = iter([100.0, 111.0 if elapsed else 100.0, 111.0 if elapsed else 100.0])+ after = 111.0 if elapsed else 104.0+ now = iter([100.0, after, after, after])events = []timeouts = []+ connections = []def connect(**kwargs):timeouts.append(kwargs)- return _mysql_test_connection(events, rows=((1,),))+ connection = _mysql_test_connection(events, rows=((1,),))+ connections.append(connection)+ return connectionmonkeypatch.setattr(m.time, "monotonic", lambda: next(now))monkeypatch.setitem(@@ -10206,6 +10286,9 @@ def test_rds_mysql_control_query_uses_remaining_deadline("read_timeout": 10.0,"write_timeout": 10.0,}]+ if not elapsed:+ assert connections[0]._read_timeout == 6.0+ assert connections[0]._write_timeout == 6.0assert events[-2:] == ["cursor.close", "connection.close"]@@ -10318,6 +10401,29 @@ def test_rds_mysql_quiescence_orders_normal_xa_normal_before_gtid(monkeypatch):]+def test_rds_mysql_quiescence_preserves_shared_deadline(monkeypatch):+ from ministack.services import rds as m++ deadlines = []+ normal = iter([(), ()])++ def result(value):+ return lambda _cluster, *, deadline=None: (+ deadlines.append(deadline) or (next(value) if hasattr(value, "__next__") else value)+ )++ _patch_mysql_quiescence(+ monkeypatch, m,+ fence=result(True), normal=result(normal), xa=result(True),+ gtid=result("source:1-7"),+ )++ assert m._wait_for_mysql_writer_quiescence(+ {}, 30, deadline=130.0,+ ) == "source:1-7"+ assert deadlines == [130.0] * 7++@pytest.mark.parametrize(("normal_results", "xa_result"), [((None,), True), (((), None), True), (((), ()), None),])@@ -10424,6 +10530,330 @@ def test_rds_mysql_quiescence_epoch_change_suppresses_result(monkeypatch):assert m._wait_for_mysql_writer_quiescence(cluster, 1) is None+def _mysql_switchover_unit_topology(monkeypatch, m):+ topology = _mysql_replication_unit_topology()+ writer, target, writer_member, target_member, global_cluster = topology+ for index, cluster in enumerate((writer, target), start=1):+ cluster.update({+ "_shared_storage_initialized": True,+ "_shared_container_id": f"container-{index}",+ "_shared_container_epoch": index,+ "Status": "available",+ "DBClusterMembers": [{"DBInstanceIdentifier": f"instance-{index}"}],+ })+ target["_mysql_replication_source_arn"] = writer["DBClusterArn"]+ global_cluster.update({+ "Engine": "aurora-mysql",+ "EngineVersion": writer["EngineVersion"],+ "GlobalClusterArn": "arn:aws:rds::111111111111:global-cluster:global-repl",+ "GlobalClusterResourceId": "cluster-globalrepl",+ "Status": "available",+ })+ clusters = {+ value: cluster+ for cluster in (writer, target)+ for value in (cluster["DBClusterIdentifier"], cluster["DBClusterArn"])+ }+ _patch_mysql_replication_unit_topology(monkeypatch, m, topology)+ monkeypatch.setattr(+ m, "_resolve_global_cluster",+ lambda identifier: global_cluster if identifier == "global-repl" else None,+ )+ monkeypatch.setattr(m, "_resolve_cluster", clusters.get)+ monkeypatch.setattr(m, "_resolve_cluster_in_request_region", clusters.get)+ monkeypatch.setattr(+ m, "_cluster_member_instances",+ lambda cluster: [{+ "DBInstanceIdentifier": cluster["DBClusterMembers"][0]["DBInstanceIdentifier"],+ "DBInstanceStatus": "available",+ }] if cluster.get("DBClusterMembers") else [],+ )+ monkeypatch.setattr(m, "_mysql_global_writer_switch_owners", {})+ return topology+++def _mysql_switchover_params(target="secondary"):+ return {+ "GlobalClusterIdentifier": "global-repl",+ "TargetDbClusterIdentifier": target,+ }+++def test_rds_mysql_gtid_wait_recomputes_shared_deadline(monkeypatch):+ from ministack.services import rds as m++ now = [100.0]+ observed = {}+ monkeypatch.setattr(m.time, "monotonic", lambda: now[0])++ class Cursor:+ def __init__(self, connection):+ self.connection = connection++ def execute(self, statement, params):+ observed["sql"] = (statement, params)++ def fetchone(self):+ return (0,)++ def close(self):+ pass++ class Connection:+ def cursor(self):+ now[0] += 4+ return Cursor(self)++ def close(self):+ pass++ def connect(_cluster, *, timeout):+ observed["connect_timeout"] = timeout+ return Connection()++ monkeypatch.setattr(m, "_mysql_replication_connection", connect)++ assert m._wait_for_mysql_gtid(+ {"DBClusterIdentifier": "secondary"}, "source:1-9", 30,+ deadline=130.0,+ ) is True+ assert observed["connect_timeout"] == 30.0+ assert observed["sql"][1] == ("source:1-9", 26.0)++[email protected]("mutation", ["password", "topology"])+def test_rds_active_switchover_rejects_mutation(monkeypatch, mutation):+ from ministack.services import rds as m++ writer, target, writer_member, target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )+ with m._shared_container_lock:+ owner = m._claim_mysql_global_writer_switch(global_cluster, writer, target)+ if mutation == "password":+ result = m._modify_db_cluster({+ "DBClusterIdentifier": "primary",+ "MasterUserPassword": "changed-password",+ })+ elif mutation == "topology":+ result = m._remove_from_global_cluster({+ "GlobalClusterIdentifier": "global-repl",+ "DbClusterIdentifier": "secondary",+ })+ else:+ result = m._failover_global_cluster({+ "GlobalClusterIdentifier": "global-repl",+ "TargetDbClusterIdentifier": "secondary",+ "AllowDataLoss": "true",+ })++ assert result[0] == 400+ assert m._active_mysql_global_writer_switch(global_cluster) is owner+ assert writer_member["IsWriter"] is True+ assert target_member["IsWriter"] is False+ assert writer.get("_MasterUserPassword") != "changed-password"+ assert target["GlobalClusterIdentifier"] == "global-repl"+ assert target["_mysql_replication_source_arn"] == writer["DBClusterArn"]+++ ("handler_name", "impl_name"),+ [+ ("_create_db_instance", "_create_db_instance_impl"),+ ("_delete_db_instance", "_delete_db_instance_impl"),+ ],+)+def test_rds_instance_mutation_prevents_switchover_claim(+ monkeypatch,+ handler_name,+ impl_name,+):+ from ministack.services import rds as m++ writer, target, _writer_member, _target_member, global_cluster = (+ _mysql_switchover_unit_topology(monkeypatch, m)+ )+ entered = threading.Event()+ release = threading.Event()+ claimed = threading.Event()+ order = []++ def mutation_impl(_params):+ entered.set()+ assert release.wait(timeout=1)+ order.append("mutation")+ return "mutated"++ monkeypatch.setattr(m, impl_name, mutation_impl)+ if handler_name == "_create_db_instance":+ params = {"DBClusterIdentifier": "primary"}+ else:+ m._instances["mutating-instance"] = {+ "DBInstanceIdentifier": "mutating-instance",+ "DBClusterIdentifier": "primary",+ }+ params = {"DBInstanceIdentifier": "mutating-instance"}+ mutation = threading.Thread(target=getattr(m, handler_name), args=(params,))+ mutation.start()+ assert entered.wait(timeout=1)++ def claim():+ with m._shared_container_lock:+ owner = m._claim_mysql_global_writer_switch(+ global_cluster, writer, target,+ )+ order.append(("claim", owner))+ claimed.set()++ claimant = threading.Thread(target=claim)+ claimant.start()+ assert claimed.wait(timeout=1)+ assert order == [("claim", None)]+ release.set()+ mutation.join(timeout=1)+ claimant.join(timeout=1)++ assert not mutation.is_alive() and not claimant.is_alive()+ assert order == [("claim", None), "mutation"]+ with m._shared_container_lock:+ assert m._claim_mysql_global_writer_switch(+ global_cluster, writer, target,+ ) is not None+++def test_rds_mysql_gtid_wait_and_write_enable_use_exact_sql(monkeypatch):+ from ministack.services import rds as m++ events = []+ rows = iter([(0,), (0, 0)])++ class Cursor:+ def execute(self, statement, params=None):+ events.append((statement, params))++ def fetchone(self):+ return next(rows)++ def close(self):+ events.append("cursor-close")++ class Connection:+ def cursor(self):+ return Cursor()++ def close(self):+ events.append("connection-close")++ monkeypatch.setattr(m, "_mysql_replication_connection", lambda *_a, **_k: Connection())+ cluster = {"DBClusterIdentifier": "secondary"}+ assert m._wait_for_mysql_gtid(cluster, "source:1-9", 10) is True+ assert m._set_mysql_cluster_writable(cluster) is True+ assert events == [+ ("SELECT WAIT_FOR_EXECUTED_GTID_SET(%s, %s)", ("source:1-9", 10.0)),+ "cursor-close", "connection-close",+ ("SET GLOBAL super_read_only=OFF", None),+ ("SET GLOBAL read_only=OFF", None),+ ("SELECT @@GLOBAL.read_only, @@GLOBAL.super_read_only", None),+ "cursor-close", "connection-close",+ ]+++def test_rds_mysql_write_enable_refences_partial_failure(monkeypatch):+ from ministack.services import rds as m++ statements = []++ class Cursor:+ def execute(self, statement, _params=None):+ statements.append(statement)+ if statement == "SET GLOBAL read_only=OFF":+ raise RuntimeError("read-only write failed")++ def close(self):+ pass++ class Connection:+ def cursor(self):+ return Cursor()++ def close(self):+ pass++ monkeypatch.setattr(m, "_mysql_replication_connection", lambda *_a, **_k: Connection())++ assert m._set_mysql_cluster_writable({"DBClusterIdentifier": "target"}) is False+ assert statements == [+ "SET GLOBAL super_read_only=OFF",+ "SET GLOBAL read_only=OFF",+ "SET GLOBAL super_read_only=ON",+ ]+++def test_rds_mysql_write_enable_refences_lost_disable_acknowledgment(monkeypatch):+ from ministack.services import rds as m++ statements = []+ server_state = {"super_read_only": True}++ class Cursor:+ def execute(self, statement, _params=None):+ statements.append(statement)+ if statement == "SET GLOBAL super_read_only=OFF":+ server_state["super_read_only"] = False+ raise RuntimeError("write applied but acknowledgment was lost")+ if statement == "SET GLOBAL super_read_only=ON":+ server_state["super_read_only"] = True++ def close(self):+ pass++ class Connection:+ def cursor(self):+ return Cursor()++ def close(self):+ pass++ monkeypatch.setattr(m, "_mysql_replication_connection", lambda *_a, **_k: Connection())++ assert m._set_mysql_cluster_writable({"DBClusterIdentifier": "target"}) is False+ assert statements == [+ "SET GLOBAL super_read_only=OFF",+ "SET GLOBAL super_read_only=ON",+ ]+ assert server_state["super_read_only"] is True++[email protected]("close_failure", ["cursor", "connection"])+def test_rds_mysql_verified_write_enable_survives_cleanup_failure(+ monkeypatch,+ close_failure,+):+ from ministack.services import rds as m++ class Cursor:+ def execute(self, _statement, _params=None):+ pass++ def fetchone(self):+ return (0, 0)++ def close(self):+ if close_failure == "cursor":+ raise RuntimeError("cursor close failed")++ class Connection:+ def cursor(self):+ return Cursor()++ def close(self):+ if close_failure == "connection":+ raise RuntimeError("connection close failed")++ monkeypatch.setattr(m, "_mysql_replication_connection", lambda *_a, **_k: Connection())+ assert m._set_mysql_cluster_writable({"DBClusterIdentifier": "target"}) is True++def _wait_for_replica_status(endpoint, timeout=120):import pymysql