Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

feat(rds): add Aurora MySQL promotion foundations (#1606)

Ian Oberstd6bcf2b8b921
diff --git a/ministack/services/rds.py b/ministack/services/rds.py
index 324738e4..c10a339d 100644
--- a/ministack/services/rds.py
+++ b/ministack/services/rds.py
@@ -120,6 +120,9 @@ _port_counter = [BASE_PORT]
_docker = None
_ministack_network = None
_shared_container_lock = threading.RLock()
+_mysql_global_writer_switch_lock = threading.RLock()
+_mysql_global_writer_switch_owners = {}
+_mysql_global_instance_mutations = {}
_MYSQL_REPLICATION_USER = "rdsrepladmin"
_MYSQL_REPLICATION_PASSWORD = "ministack-rds-replication"
@@ -2407,11 +2410,11 @@ def _mysql_cluster_connection(cluster, user, password, *, timeout=None):
)
-def _mysql_admin_connection(cluster):
- return _mysql_cluster_connection(
- cluster,
- "root",
- cluster.get("_MasterUserPassword", "password"),
+def _mysql_admin_connection(cluster, *, timeout=None):
+ args = (cluster, "root", cluster.get("_MasterUserPassword", "password"))
+ return (
+ _mysql_cluster_connection(*args)
+ if timeout is None else _mysql_cluster_connection(*args, timeout=timeout)
)
@@ -2505,9 +2508,41 @@ def _ensure_mysql_compatibility(
def _mysql_replication_connection(cluster, *, timeout=None):
args = (cluster, _MYSQL_CONTROL_USER, _MYSQL_CONTROL_PASSWORD)
- if timeout is None:
- return _mysql_cluster_connection(*args)
- return _mysql_cluster_connection(*args, timeout=timeout)
+ return (
+ _mysql_cluster_connection(*args)
+ if timeout is None else _mysql_cluster_connection(*args, timeout=timeout)
+ )
+
+
+def _mysql_deadline_timeout(*, timeout=None, deadline=None):
+ if deadline is None:
+ return timeout
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise TimeoutError("MySQL operation deadline elapsed")
+ return remaining
+
+
+def _connect_mysql(connect, cluster, timeout):
+ return connect(cluster) if timeout is None else connect(cluster, timeout=timeout)
+
+
+def _execute_mysql(cur, statement, params=None, *, deadline=None):
+ if deadline is not None:
+ timeout = _mysql_deadline_timeout(deadline=deadline)
+ cur.connection._read_timeout = timeout
+ cur.connection._write_timeout = timeout
+ return cur.execute(statement) if params is None else cur.execute(statement, params)
+
+
+def _acquire_mysql_lifecycle_lock(deadline):
+ remaining = deadline - time.monotonic()
+ if remaining <= 0 or not _shared_container_lock.acquire(timeout=remaining):
+ return False
+ if time.monotonic() <= deadline:
+ return True
+ _shared_container_lock.release()
+ return False
def _close_mysql_resources(operation, **resources):
@@ -2526,11 +2561,17 @@ def _close_mysql_resources(operation, **resources):
return succeeded
-def _ensure_mysql_control_user(cluster, *, require_quiescence_grants=False):
+def _ensure_mysql_control_user(
+ cluster, *, require_quiescence_grants=False, timeout=None, deadline=None,
+):
"""Create a local, non-replicated account for replica lifecycle SQL."""
- if cluster.get("_mysql_control_user_ready") and (
+ if (
+ cluster.get("_mysql_control_user_ready")
+ and cluster.get("_mysql_replication_status_grant_ready")
+ and (
not require_quiescence_grants
or cluster.get("_mysql_writer_quiescence_grants_ready")
+ )
):
return True
cluster_id = cluster.get("DBClusterIdentifier", "unknown")
@@ -2538,26 +2579,35 @@ def _ensure_mysql_control_user(cluster, *, require_quiescence_grants=False):
cur = None
succeeded = False
try:
- conn = _mysql_admin_connection(cluster)
+ conn = _connect_mysql(
+ _mysql_admin_connection, cluster,
+ _mysql_deadline_timeout(timeout=timeout, deadline=deadline),
+ )
if conn is None:
return False
cur = conn.cursor()
- cur.execute("SET SESSION sql_log_bin=0")
- cur.execute(
+ _execute_mysql(cur, "SET SESSION sql_log_bin=0", deadline=deadline)
+ _execute_mysql(
+ cur,
"CREATE USER IF NOT EXISTS %s@'%%' "
"IDENTIFIED WITH mysql_native_password BY %s",
(_MYSQL_CONTROL_USER, _MYSQL_CONTROL_PASSWORD),
+ deadline=deadline,
)
- cur.execute(
- "GRANT PROCESS, RELOAD ON *.* TO %s@'%%'",
+ _execute_mysql(
+ cur,
+ "GRANT PROCESS, RELOAD, REPLICATION CLIENT ON *.* TO %s@'%%'",
(_MYSQL_CONTROL_USER,),
+ deadline=deadline,
)
- cur.execute(
+ _execute_mysql(
+ cur,
"GRANT CONNECTION_ADMIN, REPLICATION_SLAVE_ADMIN, "
"SYSTEM_VARIABLES_ADMIN, XA_RECOVER_ADMIN ON *.* TO %s@'%%'",
(_MYSQL_CONTROL_USER,),
+ deadline=deadline,
)
- cur.execute("FLUSH PRIVILEGES")
+ _execute_mysql(cur, "FLUSH PRIVILEGES", deadline=deadline)
succeeded = True
except Exception as e:
logger.warning(
@@ -2570,17 +2620,20 @@ def _ensure_mysql_control_user(cluster, *, require_quiescence_grants=False):
succeeded = False
if succeeded:
cluster["_mysql_control_user_ready"] = True
+ cluster["_mysql_replication_status_grant_ready"] = True
cluster["_mysql_writer_quiescence_grants_ready"] = True
return succeeded
-def _set_mysql_writer_fence(conn):
+def _set_mysql_writer_fence(conn, *, deadline=None):
"""Fence on a caller-owned connection; never close or bound that owner."""
cur = None
succeeded = False
try:
cur = conn.cursor()
- cur.execute("SET GLOBAL super_read_only=ON")
+ _execute_mysql(
+ cur, "SET GLOBAL super_read_only=ON", deadline=deadline,
+ )
succeeded = True
except Exception as e:
logger.warning("RDS: failed to install MySQL writer fence: %s", e)
@@ -2617,15 +2670,11 @@ def _mysql_control_query(cluster, statement, *, deadline=None):
timeout = deadline - time.monotonic()
if timeout <= 0:
return None
- conn = (
- _mysql_replication_connection(cluster)
- if timeout is None
- else _mysql_replication_connection(cluster, timeout=timeout)
- )
+ conn = _connect_mysql(_mysql_replication_connection, cluster, timeout)
if conn is None:
return None
cur = conn.cursor()
- cur.execute(statement)
+ _execute_mysql(cur, statement, deadline=deadline)
result = cur.fetchall()
if isinstance(result, (tuple, list)) and (
deadline is None or time.monotonic() <= deadline
@@ -2712,7 +2761,9 @@ def _capture_mysql_gtid_executed(cluster, *, deadline=None):
return gtid if isinstance(gtid, str) and gtid and gtid == gtid.strip() else None
-def _wait_for_mysql_writer_quiescence(cluster, timeout, poll_interval=0.1):
+def _wait_for_mysql_writer_quiescence(
+ cluster, timeout, poll_interval=0.1, *, deadline=None,
+):
"""Naturally drain a separately fenced writer and capture its GTID set."""
if (
isinstance(timeout, bool)
@@ -2733,7 +2784,14 @@ def _wait_for_mysql_writer_quiescence(cluster, timeout, poll_interval=0.1):
or poll_interval <= 0
):
return None
- deadline = time.monotonic() + timeout
+ if deadline is None:
+ deadline = time.monotonic() + timeout
+ elif (
+ isinstance(deadline, bool)
+ or not isinstance(deadline, (int, float))
+ or not math.isfinite(deadline)
+ ):
+ return None
container_epoch = cluster.get("_shared_container_epoch")
def _retry():
@@ -2795,42 +2853,149 @@ def _wait_for_mysql_writer_quiescence(cluster, timeout, poll_interval=0.1):
return gtid
-def _ensure_mysql_replication_user(cluster):
+def _wait_for_mysql_gtid(cluster, gtid, timeout, *, deadline=None):
+ if (
+ not isinstance(gtid, str)
+ or not gtid
+ or gtid != gtid.strip()
+ or isinstance(timeout, bool)
+ or not isinstance(timeout, (int, float))
+ ):
+ return False
+ try:
+ timeout = float(timeout)
+ except (OverflowError, ValueError):
+ return False
+ if not math.isfinite(timeout) or timeout <= 0:
+ return False
+
+ conn = None
+ cur = None
+ succeeded = False
+ try:
+ conn = _connect_mysql(
+ _mysql_replication_connection, cluster,
+ _mysql_deadline_timeout(timeout=timeout, deadline=deadline),
+ )
+ if conn is None:
+ return False
+ cur = conn.cursor()
+ wait_timeout = _mysql_deadline_timeout(timeout=timeout, deadline=deadline)
+ _execute_mysql(
+ cur,
+ "SELECT WAIT_FOR_EXECUTED_GTID_SET(%s, %s)",
+ (gtid, wait_timeout),
+ deadline=deadline,
+ )
+ row = cur.fetchone()
+ succeeded = bool(
+ isinstance(row, (tuple, list))
+ and len(row) == 1
+ and type(row[0]) is int
+ and row[0] == 0
+ )
+ except Exception as e:
+ logger.warning(
+ "RDS: failed waiting for MySQL GTID convergence on %s: %s",
+ cluster.get("DBClusterIdentifier", "unknown"), e,
+ )
+ finally:
+ if not _close_mysql_resources("GTID wait", cursor=cur, connection=conn):
+ succeeded = False
+ return succeeded
+
+
+def _set_mysql_cluster_writable(cluster, *, timeout=None, deadline=None):
+ """Enable and verify writes, restoring the fence on partial failure."""
+ conn = None
+ cur = None
+ super_read_only_disable_attempted = False
+ succeeded = False
+ try:
+ conn = _connect_mysql(
+ _mysql_replication_connection, cluster,
+ _mysql_deadline_timeout(timeout=timeout, deadline=deadline),
+ )
+ if conn is None:
+ return False
+ cur = conn.cursor()
+ super_read_only_disable_attempted = True
+ _execute_mysql(cur, "SET GLOBAL super_read_only=OFF", deadline=deadline)
+ _execute_mysql(cur, "SET GLOBAL read_only=OFF", deadline=deadline)
+ _execute_mysql(
+ cur, "SELECT @@GLOBAL.read_only, @@GLOBAL.super_read_only",
+ deadline=deadline,
+ )
+ row = cur.fetchone()
+ succeeded = bool(
+ isinstance(row, (tuple, list))
+ and len(row) == 2
+ and all(type(value) is int for value in row)
+ and tuple(row) == (0, 0)
+ )
+ except Exception as e:
+ logger.warning(
+ "RDS: failed enabling MySQL writes on %s: %s",
+ cluster.get("DBClusterIdentifier", "unknown"), e,
+ )
+ finally:
+ if not succeeded and super_read_only_disable_attempted and cur is not None:
+ try:
+ # Safety compensation uses the connection's last bounded timeout.
+ cur.execute("SET GLOBAL super_read_only=ON")
+ except Exception as e:
+ logger.warning(
+ "RDS: failed restoring MySQL fence on %s: %s",
+ cluster.get("DBClusterIdentifier", "unknown"), e,
+ )
+ _close_mysql_resources("write enable", cursor=cur, connection=conn)
+ return succeeded
+
+
+def _ensure_mysql_replication_user(cluster, *, timeout=None, deadline=None):
"""Ensure the internal replication account exists on a writer cluster."""
cluster_id = cluster.get("DBClusterIdentifier", "unknown")
conn = None
cur = None
+ succeeded = False
try:
- conn = _mysql_admin_connection(cluster)
+ conn = _connect_mysql(
+ _mysql_admin_connection, cluster,
+ _mysql_deadline_timeout(timeout=timeout, deadline=deadline),
+ )
if conn is None:
return False
cur = conn.cursor()
- cur.execute(
+ _execute_mysql(
+ cur,
"CREATE USER IF NOT EXISTS %s@'%%' "
"IDENTIFIED WITH mysql_native_password BY %s",
(_MYSQL_REPLICATION_USER, _MYSQL_REPLICATION_PASSWORD),
+ deadline=deadline,
)
- cur.execute(
+ _execute_mysql(
+ cur,
"GRANT REPLICATION SLAVE ON *.* TO %s@'%%'",
(_MYSQL_REPLICATION_USER,),
+ deadline=deadline,
)
- cur.execute("FLUSH PRIVILEGES")
- return True
+ _execute_mysql(cur, "FLUSH PRIVILEGES", deadline=deadline)
+ succeeded = True
except Exception as e:
logger.warning(
"RDS: failed to prepare MySQL replication source %s: %s",
cluster_id,
e,
)
- return False
finally:
- if cur is not None:
- cur.close()
- if conn is not None:
- conn.close()
+ if not _close_mysql_resources("replication user", cursor=cur, connection=conn):
+ succeeded = False
+ return succeeded
-def _configure_mysql_replica_from_source(cluster_id, replica, source):
+def _configure_mysql_replica_from_source(
+ cluster_id, replica, source, *, timeout=None, deadline=None,
+):
"""Configure a replica channel from an explicit source cluster.
The caller owns role validation and must ensure the replication account is
@@ -2842,13 +3007,17 @@ def _configure_mysql_replica_from_source(cluster_id, replica, source):
"""
conn = None
cur = None
+ succeeded = False
try:
- conn = _mysql_replication_connection(replica)
+ conn = _connect_mysql(
+ _mysql_replication_connection, replica,
+ _mysql_deadline_timeout(timeout=timeout, deadline=deadline),
+ )
if conn is None:
- return False
+ raise RuntimeError("MySQL replication connection is unavailable")
cur = conn.cursor()
try:
- cur.execute("STOP REPLICA")
+ _execute_mysql(cur, "STOP REPLICA", deadline=deadline)
except Exception as e:
logger.debug(
"RDS: STOP REPLICA was a no-op for %s: %s",
@@ -2860,8 +3029,9 @@ def _configure_mysql_replica_from_source(cluster_id, replica, source):
# and relay logs but deliberately preserves gtid_executed. Within
# the source's configured retention window, auto-position can
# request every transaction not yet applied before the restart.
- cur.execute("RESET REPLICA ALL")
- cur.execute(
+ _execute_mysql(cur, "RESET REPLICA ALL", deadline=deadline)
+ _execute_mysql(
+ cur,
"CHANGE REPLICATION SOURCE TO "
"SOURCE_HOST=%s, SOURCE_PORT=%s, SOURCE_USER=%s, "
"SOURCE_PASSWORD=%s, SOURCE_AUTO_POSITION=1, "
@@ -2872,21 +3042,25 @@ def _configure_mysql_replica_from_source(cluster_id, replica, source):
_MYSQL_REPLICATION_USER,
_MYSQL_REPLICATION_PASSWORD,
),
+ deadline=deadline,
)
- cur.execute("START REPLICA")
- cur.execute("SET GLOBAL super_read_only=ON")
+ _execute_mysql(cur, "START REPLICA", deadline=deadline)
+ _execute_mysql(cur, "SET GLOBAL super_read_only=ON", deadline=deadline)
+ succeeded = True
except Exception as e:
logger.warning(
"RDS: failed to configure MySQL replication for %s: %s",
cluster_id,
e,
)
- return False
finally:
- if cur is not None:
- cur.close()
- if conn is not None:
- conn.close()
+ if not _close_mysql_resources(
+ "replication configuration", cursor=cur, connection=conn,
+ ):
+ succeeded = False
+
+ if not succeeded:
+ return False
replica["_mysql_replication_source_arn"] = source.get("DBClusterArn")
replica.pop("_mysql_replication_reset_pending", None)
@@ -2912,6 +3086,8 @@ def _configure_mysql_replication(cluster_id, cluster):
"""
if not _aurora_mysql_8_replication_enabled(cluster):
return None
+ if _mysql_global_writer_switch_mutation_error(cluster):
+ return False
global_cluster, member = _global_cluster_member_for_cluster(cluster)
if not global_cluster or not member:
@@ -2974,6 +3150,9 @@ def _reset_mysql_replication_channel(
cluster,
*,
clear_super_read_only=True,
+ allow_missing_channel=False,
+ timeout=None,
+ deadline=None,
):
"""Stop and reset a replica channel without consulting global role metadata.
@@ -2983,12 +3162,18 @@ def _reset_mysql_replication_channel(
source/detach metadata for its caller to clear and, when requested, clears
only ``super_read_only`` to preserve existing detach behavior. It does not
guarantee general-client writability while ``read_only`` remains enabled.
+ Callers may explicitly tolerate an already-absent channel; that case is
+ verified before destructive channel SQL and recorded as reset.
"""
if not cluster.get("_shared_storage_initialized"):
return True
if not cluster.get("_shared_container_ready"):
return False
- if not _ensure_mysql_control_user(cluster):
+ control_ready = (
+ _ensure_mysql_control_user(cluster)
+ if deadline is None else _ensure_mysql_control_user(cluster, deadline=deadline)
+ )
+ if not control_ready:
return False
# Persist the transition before destructive SQL. A failed request keeps
@@ -3000,23 +3185,34 @@ def _reset_mysql_replication_channel(
conn = None
cur = None
try:
- conn = _mysql_replication_connection(cluster)
+ conn = _connect_mysql(
+ _mysql_replication_connection, cluster,
+ _mysql_deadline_timeout(timeout=timeout, deadline=deadline),
+ )
if conn is None:
return False
cur = conn.cursor()
- if state == "requested":
- cur.execute("STOP REPLICA")
- cluster["_mysql_replication_detach_state"] = "stopped"
- state = "stopped"
- if state in ("stopped", "resetting"):
- # ``resetting`` is written first so a crash between MySQL applying
- # RESET and Python publishing the next state is recoverable.
- cluster["_mysql_replication_detach_state"] = "resetting"
- cur.execute("RESET REPLICA ALL")
- cluster["_mysql_replication_detach_state"] = "reset"
- state = "reset"
+ if allow_missing_channel:
+ _execute_mysql(cur, "SHOW REPLICA STATUS", deadline=deadline)
+ if cur.fetchone() is None:
+ cluster["_mysql_replication_detach_state"] = "reset"
+ state = "reset"
+ if state != "reset":
+ if state == "requested":
+ _execute_mysql(cur, "STOP REPLICA", deadline=deadline)
+ cluster["_mysql_replication_detach_state"] = "stopped"
+ state = "stopped"
+ if state in ("stopped", "resetting"):
+ # ``resetting`` is written first so a crash between MySQL applying
+ # RESET and Python publishing the next state is recoverable.
+ cluster["_mysql_replication_detach_state"] = "resetting"
+ _execute_mysql(cur, "RESET REPLICA ALL", deadline=deadline)
+ cluster["_mysql_replication_detach_state"] = "reset"
+ state = "reset"
if state == "reset" and clear_super_read_only:
- cur.execute("SET GLOBAL super_read_only=OFF")
+ _execute_mysql(
+ cur, "SET GLOBAL super_read_only=OFF", deadline=deadline,
+ )
except Exception as e:
logger.warning(
"RDS: failed to reset MySQL replication channel for %s: %s",
@@ -3025,10 +3221,10 @@ def _reset_mysql_replication_channel(
)
raise
finally:
- if cur is not None:
- cur.close()
- if conn is not None:
- conn.close()
+ if not _close_mysql_resources(
+ "replication reset", cursor=cur, connection=conn,
+ ):
+ raise RuntimeError("MySQL replication reset cleanup failed")
return True
@@ -3846,7 +4042,16 @@ def _unregister_instance_from_clusters(db_id):
# ---------------------------------------------------------------------------
def _create_db_instance(p):
- return _create_db_instance_impl(p)
+ cluster = None
+ cluster_id = _p(p, "DBClusterIdentifier")
+ cluster = _resolve_cluster_in_request_region(cluster_id) if cluster_id else None
+ if cluster and (switch_error := _begin_mysql_global_instance_mutation(cluster)):
+ return switch_error
+ try:
+ return _create_db_instance_impl(p)
+ finally:
+ if cluster:
+ _end_mysql_global_instance_mutation(cluster)
def _create_db_instance_impl(p):
@@ -4472,6 +4677,22 @@ def _create_db_instance_impl(p):
def _delete_db_instance(p):
+ cluster = None
+ instance = _resolve_instance(_p(p, "DBInstanceIdentifier"))
+ cluster_id = instance and (
+ instance.get("_shared_cluster_id") or instance.get("DBClusterIdentifier")
+ )
+ cluster = _resolve_cluster_in_request_region(cluster_id) if cluster_id else None
+ if cluster and (switch_error := _begin_mysql_global_instance_mutation(cluster)):
+ return switch_error
+ try:
+ return _delete_db_instance_impl(p)
+ finally:
+ if cluster:
+ _end_mysql_global_instance_mutation(cluster)
+
+
+def _delete_db_instance_impl(p):
db_id = _p(p, "DBInstanceIdentifier")
instance = _resolve_instance(db_id)
if not instance:
@@ -4917,6 +5138,11 @@ def _serverlessv2_from_params(p):
def _create_db_cluster(p):
+ with _shared_container_lock:
+ return _create_db_cluster_impl(p)
+
+
+def _create_db_cluster_impl(p):
cluster_id = _p(p, "DBClusterIdentifier")
if not cluster_id:
return _error("MissingParameter", "DBClusterIdentifier is required", 400)
@@ -4932,6 +5158,11 @@ def _create_db_cluster(p):
if global_cluster_id and not global_cluster:
return _error("GlobalClusterNotFoundFault",
f"Global cluster {global_cluster_id} not found.", 404)
+ if global_cluster:
+ if switch_error := _mysql_global_writer_switch_mutation_error(
+ _global_cluster_writer_cluster(global_cluster),
+ ):
+ return switch_error
if global_cluster and _global_cluster_member_in_request_region(global_cluster):
return _error(
"InvalidParameterValue",
@@ -5393,6 +5624,11 @@ def _apply_cluster_master_password_change(cluster, new_pass):
def _modify_db_cluster(p):
+ with _shared_container_lock:
+ return _modify_db_cluster_impl(p)
+
+
+def _modify_db_cluster_impl(p):
cluster_id = _p(p, "DBClusterIdentifier")
cluster = _resolve_cluster_in_request_region(cluster_id)
if not cluster:
@@ -5400,6 +5636,8 @@ def _modify_db_cluster(p):
if wrong_region:
return wrong_region
return _error("DBClusterNotFoundFault", f"DBCluster {cluster_id} not found.", 404)
+ if switch_error := _mysql_global_writer_switch_mutation_error(cluster):
+ return switch_error
# Validate the password parameters together before mutating anything, so
# a rejected combination cannot leave a half-applied password change.
@@ -6104,10 +6342,18 @@ def _modify_subnet_group(p):
def _cluster_member_instances(cluster):
"""Resolve a cluster's member records to their instance dicts."""
+ parsed = _parse_rds_arn(cluster.get("DBClusterArn", ""))
+ scope = parsed[0] if parsed else None
+
+ def resolve(instance_id):
+ return (
+ _instances.get_scoped(scope.account_id, scope.region, instance_id)
+ if scope else _instances.get(instance_id)
+ )
return [
inst
for inst in (
- _instances.get(member.get("DBInstanceIdentifier"))
+ resolve(member.get("DBInstanceIdentifier"))
for member in cluster.get("DBClusterMembers", [])
)
if inst is not None
@@ -6497,6 +6743,11 @@ def _stop_db_cluster(p):
def _failover_db_cluster(p):
+ with _shared_container_lock:
+ return _failover_db_cluster_impl(p)
+
+
+def _failover_db_cluster_impl(p):
"""Force an intra-cluster failover: promote a reader member to writer.
Metadata-only for now, mirroring ``_failover_global_cluster``: the member
@@ -6522,6 +6773,8 @@ def _failover_db_cluster(p):
if wrong_region:
return wrong_region
return _error("DBClusterNotFoundFault", f"DBCluster {cluster_id} not found.", 404)
+ if switch_error := _mysql_global_writer_switch_mutation_error(cluster):
+ return switch_error
cluster_id = cluster.get("DBClusterIdentifier", cluster_id)
status = cluster.get("Status")
if status != "available":
@@ -6911,6 +7164,15 @@ def _describe_global_clusters(p):
gcs = [gc]
else:
gcs = list(_global_clusters.values())
+ with _shared_container_lock:
+ snapshots = []
+ for gc in gcs:
+ switching = bool(_active_mysql_global_writer_switch(gc))
+ snapshot = copy.deepcopy(gc)
+ if switching:
+ snapshot["Status"] = "switching-over"
+ snapshots.append(snapshot)
+ gcs = snapshots
members_xml = "".join(
f"<GlobalCluster>{_global_cluster_xml(gc)}</GlobalCluster>" for gc in gcs
@@ -6957,6 +7219,8 @@ def _remove_from_global_cluster(p):
with _shared_container_lock:
members = gc.get("GlobalClusterMembers", [])
cluster = _resolve_cluster(db_cluster_id)
+ if switch_error := _mysql_global_writer_switch_mutation_error(cluster):
+ return switch_error
db_cluster_arn = cluster["DBClusterArn"] if cluster else db_cluster_id
member = next(
(m for m in members if m["DBClusterArn"] == db_cluster_arn),
@@ -7005,6 +7269,11 @@ def _remove_from_global_cluster(p):
def _modify_global_cluster(p):
+ with _shared_container_lock:
+ return _modify_global_cluster_impl(p)
+
+
+def _modify_global_cluster_impl(p):
gc_id = _p(p, "GlobalClusterIdentifier")
invalid_id = _invalid_global_cluster_identifier_error(gc_id)
if invalid_id:
@@ -7013,6 +7282,10 @@ def _modify_global_cluster(p):
if not gc:
return _error("GlobalClusterNotFoundFault",
f"Global cluster {gc_id} not found.", 404)
+ if switch_error := _mysql_global_writer_switch_mutation_error(
+ _global_cluster_writer_cluster(gc),
+ ):
+ return switch_error
engine_version = _p(p, "EngineVersion")
engine_version_error = _unsupported_aurora_engine_version_error(
@@ -7063,6 +7336,151 @@ def _find_global_cluster_target_member(gc, target_cluster_id):
return cluster, member
+def _active_mysql_global_writer_switch(gc):
+ owner = _mysql_global_writer_switch_owners.get(id(gc)) if gc else None
+ if owner and owner.get("global_cluster") is gc:
+ return owner
+ return None
+
+
+def _mysql_global_writer_switch_mutation_error(cluster):
+ if not cluster:
+ return None
+ with _shared_container_lock:
+ return _mysql_global_writer_switch_mutation_error_unlocked(cluster)
+
+
+def _mysql_global_writer_switch_mutation_error_unlocked(cluster):
+ gc, _member = _global_cluster_member_for_cluster(cluster)
+ if not _active_mysql_global_writer_switch(gc):
+ return None
+ return _error(
+ "InvalidDBClusterStateFault",
+ f"DBCluster {cluster.get('DBClusterIdentifier', '')} cannot be "
+ "modified while a global writer switchover is in progress.",
+ 400,
+ )
+
+
+def _begin_mysql_global_instance_mutation(cluster):
+ with _mysql_global_writer_switch_lock:
+ if error := _mysql_global_writer_switch_mutation_error_unlocked(cluster):
+ return error
+ entry = _mysql_global_instance_mutations.setdefault(
+ id(cluster), {"cluster": cluster, "count": 0},
+ )
+ entry["count"] += 1
+ return None
+
+
+def _end_mysql_global_instance_mutation(cluster):
+ with _mysql_global_writer_switch_lock:
+ entry = _mysql_global_instance_mutations.get(id(cluster))
+ if not entry or entry["cluster"] is not cluster:
+ return
+ entry["count"] -= 1
+ if not entry["count"]:
+ _mysql_global_instance_mutations.pop(id(cluster), None)
+
+
+def _active_mysql_global_instance_mutation(cluster):
+ entry = _mysql_global_instance_mutations.get(id(cluster))
+ return bool(entry and entry["cluster"] is cluster and entry["count"])
+
+
+def _mysql_global_writer_switch_topology(gc):
+ return tuple(
+ (member.get("DBClusterArn"), bool(member.get("IsWriter")))
+ for member in gc.get("GlobalClusterMembers", [])
+ )
+
+
+def _mysql_global_writer_switch_identity(cluster):
+ compute = tuple(
+ (instance.get("DBInstanceIdentifier"), instance.get("DBInstanceStatus"))
+ for instance in _cluster_member_instances(cluster)
+ )
+ return (cluster.get("_shared_container_epoch"), cluster.get("_shared_container_id"), compute)
+
+
+def _mysql_global_writer_switch_ready(gc, source, target):
+ return bool(
+ gc.get("Status") == "available"
+ and source.get("EngineVersion") == target.get("EngineVersion")
+ and _mysql_gtid_history_ready(source)
+ and _mysql_gtid_history_ready(target)
+ and all(
+ cluster.get("_shared_storage_initialized")
+ and cluster.get("_shared_container_ready") is True
+ and cluster.get("Status") == "available"
+ and cluster.get("DBClusterMembers")
+ and any(
+ instance.get("DBInstanceStatus") == "available"
+ for instance in _cluster_member_instances(cluster)
+ )
+ and cluster.get("_shared_container_id")
+ and cluster.get("_shared_internal_address")
+ and cluster.get("_shared_internal_port")
+ and isinstance(cluster.get("_shared_container_epoch"), int)
+ and cluster.get("_shared_container_epoch") > 0
+ and not cluster.get("_pending_master_password_rotation")
+ and not cluster.get("_mysql_replication_detach_state")
+ and not _active_mysql_global_instance_mutation(cluster)
+ for cluster in (source, target)
+ )
+ and not source.get("_mysql_replication_source_arn")
+ and target.get("_mysql_replication_source_arn") == source.get("DBClusterArn")
+ )
+
+
+def _claim_mysql_global_writer_switch(gc, source, target):
+ with _mysql_global_writer_switch_lock:
+ if _active_mysql_global_writer_switch(gc) or any(
+ _active_mysql_global_instance_mutation(cluster)
+ for cluster in (source, target)
+ ):
+ return None
+ owner = {
+ "global_cluster": gc,
+ "source": source,
+ "target": target,
+ "topology": _mysql_global_writer_switch_topology(gc),
+ "source_identity": _mysql_global_writer_switch_identity(source),
+ "target_identity": _mysql_global_writer_switch_identity(target),
+ }
+ _mysql_global_writer_switch_owners[id(gc)] = owner
+ return owner
+
+
+def _mysql_global_writer_switch_still_owned(owner, expected_topology):
+ gc = owner["global_cluster"]
+ source = owner["source"]
+ target = owner["target"]
+ return bool(
+ _active_mysql_global_writer_switch(gc) is owner
+ and _mysql_global_writer_switch_topology(gc) == expected_topology
+ and _mysql_global_writer_switch_identity(source) == owner["source_identity"]
+ and _mysql_global_writer_switch_identity(target) == owner["target_identity"]
+ and source.get("_shared_container_ready") is True
+ and target.get("_shared_container_ready") is True
+ and source.get("_shared_storage_initialized") is True
+ and target.get("_shared_storage_initialized") is True
+ and not source.get("_pending_master_password_rotation")
+ and not target.get("_pending_master_password_rotation")
+ )
+
+
+def _finish_mysql_global_writer_switch(owner, *, repair_required=False):
+ gc = owner["global_cluster"]
+ if _active_mysql_global_writer_switch(gc) is not owner:
+ return
+ if repair_required:
+ owner["state"] = "repair_required"
+ else:
+ _mysql_global_writer_switch_owners.pop(id(gc), None)
+ gc["Status"] = "available"
+
+
def _switch_global_cluster_writer(p, *, allow_data_loss=False):
gc_id = _p(p, "GlobalClusterIdentifier")
target_cluster_id = _p(p, "TargetDbClusterIdentifier")
@@ -9338,6 +9756,8 @@ def reset():
# restore worker can pass its membership check after reset has enumerated
# resources, then create a container after the stores are cleared.
with _shared_container_lock:
+ _mysql_global_writer_switch_owners.clear()
+ _mysql_global_instance_mutations.clear()
docker_client = _get_docker()
if docker_client:
# Shared containers are cluster-owned. Reap them once from the cluster
diff --git a/tests/test_rds.py b/tests/test_rds.py
index 8cfc7f42..88ac9917 100644
--- a/tests/test_rds.py
+++ b/tests/test_rds.py
@@ -8773,7 +8773,7 @@ def test_rds_mysql_replication_restore_resets_once_then_is_idempotent(monkeypatc
def close(self):
pass
- monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda _cluster: True)
+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)
monkeypatch.setattr(
m,
"_mysql_replication_connection",
@@ -8838,7 +8838,7 @@ def test_rds_mysql_replication_writer_sweep_closes_reverse_readiness_race(
def close(self):
pass
- monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda _cluster: True)
+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)
monkeypatch.setattr(
m,
"_mysql_replication_connection",
@@ -8897,7 +8897,7 @@ def test_rds_mysql_replication_failure_is_retryable_and_closes_connections(
def close(self):
closed.append("connection")
- monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda _cluster: True)
+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)
monkeypatch.setattr(m, "_mysql_replication_connection", lambda _cluster: FakeConnection())
monkeypatch.setattr(
m,
@@ -9496,6 +9496,63 @@ def test_rds_mysql_replication_channel_reset_is_role_independent_and_keeps_fence
assert closed == ["cursor", "connection"]
+ ("clear_super_read_only", "expected_statements"),
+ [
+ (True, ["SHOW REPLICA STATUS", "SET GLOBAL super_read_only=OFF"]),
+ (False, ["SHOW REPLICA STATUS"]),
+ ],
+)
+def test_rds_mysql_replication_channel_reset_allows_missing_channel(
+ monkeypatch,
+ clear_super_read_only,
+ expected_statements,
+):
+ from ministack.services import rds as m
+
+ writer, _secondary, _writer_member, _secondary_member, _global = (
+ _mysql_replication_unit_topology()
+ )
+ writer["_shared_storage_initialized"] = True
+ statements = []
+ closed = []
+
+ class FakeCursor:
+ def execute(self, statement, params=None):
+ statements.append((statement, params))
+
+ def fetchone(self):
+ return None
+
+ def close(self):
+ closed.append("cursor")
+
+ class FakeConnection:
+ def cursor(self):
+ return FakeCursor()
+
+ def close(self):
+ closed.append("connection")
+
+ monkeypatch.setattr(m, "_ensure_mysql_control_user", lambda _cluster: True)
+ monkeypatch.setattr(
+ m,
+ "_mysql_replication_connection",
+ lambda cluster: FakeConnection() if cluster is writer else None,
+ )
+
+ assert m._reset_mysql_replication_channel(
+ "primary",
+ writer,
+ clear_super_read_only=clear_super_read_only,
+ allow_missing_channel=True,
+ ) is True
+
+ assert [statement for statement, _params in statements] == expected_statements
+ assert writer["_mysql_replication_detach_state"] == "reset"
+ assert closed == ["cursor", "connection"]
+
+
@pytest.mark.parametrize(
"failed_statement",
[
@@ -9668,7 +9725,7 @@ def test_rds_mysql_replication_detach_failure_rolls_channel_back_atomically(
def close(self):
pass
- monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda _cluster: True)
+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)
monkeypatch.setattr(m, "_mysql_replication_connection", lambda _cluster: FakeConnection())
assert m._detach_mysql_replication("secondary", secondary) is False
@@ -9725,7 +9782,7 @@ def test_rds_mysql_replication_detach_partial_rollback_retries_from_requested(
def close(self):
pass
- monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda _cluster: True)
+ monkeypatch.setattr(m, "_ensure_mysql_replication_user", lambda *_a, **_k: True)
monkeypatch.setattr(m, "_mysql_replication_connection", lambda _cluster: FakeConnection())
assert m._detach_mysql_replication("secondary", secondary) is False
@@ -10005,6 +10062,7 @@ def test_rds_mysql_control_user_is_local_and_used_for_replica_sql(monkeypatch):
cluster = {
"DBClusterIdentifier": "secondary",
"_mysql_control_user_ready": True,
+ "_mysql_replication_status_grant_ready": True,
}
events = []
connection_args = []
@@ -10025,7 +10083,7 @@ def test_rds_mysql_control_user_is_local_and_used_for_replica_sql(monkeypatch):
"CREATE USER IF NOT EXISTS %s@'%%' "
"IDENTIFIED WITH mysql_native_password BY %s"
),
- "GRANT PROCESS, RELOAD ON *.* TO %s@'%%'",
+ "GRANT PROCESS, RELOAD, REPLICATION CLIENT ON *.* TO %s@'%%'",
(
"GRANT CONNECTION_ADMIN, REPLICATION_SLAVE_ADMIN, "
"SYSTEM_VARIABLES_ADMIN, XA_RECOVER_ADMIN ON *.* TO %s@'%%'"
@@ -10033,6 +10091,7 @@ def test_rds_mysql_control_user_is_local_and_used_for_replica_sql(monkeypatch):
"FLUSH PRIVILEGES",
]
assert statements[2][1] == statements[3][1] == (m._MYSQL_CONTROL_USER,)
+ assert cluster["_mysql_replication_status_grant_ready"] is True
assert cluster["_mysql_writer_quiescence_grants_ready"] is True
assert m._ensure_mysql_control_user(cluster) is True
assert m._ensure_mysql_control_user(
@@ -10040,6 +10099,11 @@ def test_rds_mysql_control_user_is_local_and_used_for_replica_sql(monkeypatch):
) is True
assert len(events) == grant_count
+ cluster.pop("_mysql_replication_status_grant_ready")
+ assert m._ensure_mysql_control_user(cluster) is True
+ assert len(events) == grant_count * 2
+ assert cluster["_mysql_replication_status_grant_ready"] is True
+
monkeypatch.setattr(
m,
"_mysql_cluster_connection",
@@ -10078,7 +10142,9 @@ def _mysql_test_connection(events, rows=(), failure=None):
class FakeConnection:
def cursor(self):
- return FakeCursor()
+ cursor = FakeCursor()
+ cursor.connection = self
+ return cursor
def close(self):
events.append("connection.close")
@@ -10132,6 +10198,16 @@ def test_rds_mysql_writer_fence_keeps_caller_connection(monkeypatch, failure, ex
assert "connection.close" not in events
+def test_rds_mysql_writer_fence_recomputes_shared_deadline(monkeypatch):
+ from ministack.services import rds as m
+
+ conn = _mysql_test_connection([])
+ monkeypatch.setattr(m.time, "monotonic", lambda: 104.0)
+
+ assert m._set_mysql_writer_fence(conn, deadline=110.0) is True
+ assert conn._read_timeout == conn._write_timeout == 6.0
+
+
@pytest.mark.parametrize(
("readback", "expected"),
[
@@ -10182,13 +10258,17 @@ def test_rds_mysql_control_query_uses_remaining_deadline(
):
from ministack.services import rds as m
- now = iter([100.0, 111.0 if elapsed else 100.0, 111.0 if elapsed else 100.0])
+ after = 111.0 if elapsed else 104.0
+ now = iter([100.0, after, after, after])
events = []
timeouts = []
+ connections = []
def connect(**kwargs):
timeouts.append(kwargs)
- return _mysql_test_connection(events, rows=((1,),))
+ connection = _mysql_test_connection(events, rows=((1,),))
+ connections.append(connection)
+ return connection
monkeypatch.setattr(m.time, "monotonic", lambda: next(now))
monkeypatch.setitem(
@@ -10206,6 +10286,9 @@ def test_rds_mysql_control_query_uses_remaining_deadline(
"read_timeout": 10.0,
"write_timeout": 10.0,
}]
+ if not elapsed:
+ assert connections[0]._read_timeout == 6.0
+ assert connections[0]._write_timeout == 6.0
assert events[-2:] == ["cursor.close", "connection.close"]
@@ -10318,6 +10401,29 @@ def test_rds_mysql_quiescence_orders_normal_xa_normal_before_gtid(monkeypatch):
]
+def test_rds_mysql_quiescence_preserves_shared_deadline(monkeypatch):
+ from ministack.services import rds as m
+
+ deadlines = []
+ normal = iter([(), ()])
+
+ def result(value):
+ return lambda _cluster, *, deadline=None: (
+ deadlines.append(deadline) or (next(value) if hasattr(value, "__next__") else value)
+ )
+
+ _patch_mysql_quiescence(
+ monkeypatch, m,
+ fence=result(True), normal=result(normal), xa=result(True),
+ gtid=result("source:1-7"),
+ )
+
+ assert m._wait_for_mysql_writer_quiescence(
+ {}, 30, deadline=130.0,
+ ) == "source:1-7"
+ assert deadlines == [130.0] * 7
+
+
@pytest.mark.parametrize(("normal_results", "xa_result"), [
((None,), True), (((), None), True), (((), ()), None),
])
@@ -10424,6 +10530,330 @@ def test_rds_mysql_quiescence_epoch_change_suppresses_result(monkeypatch):
assert m._wait_for_mysql_writer_quiescence(cluster, 1) is None
+def _mysql_switchover_unit_topology(monkeypatch, m):
+ topology = _mysql_replication_unit_topology()
+ writer, target, writer_member, target_member, global_cluster = topology
+ for index, cluster in enumerate((writer, target), start=1):
+ cluster.update({
+ "_shared_storage_initialized": True,
+ "_shared_container_id": f"container-{index}",
+ "_shared_container_epoch": index,
+ "Status": "available",
+ "DBClusterMembers": [{"DBInstanceIdentifier": f"instance-{index}"}],
+ })
+ target["_mysql_replication_source_arn"] = writer["DBClusterArn"]
+ global_cluster.update({
+ "Engine": "aurora-mysql",
+ "EngineVersion": writer["EngineVersion"],
+ "GlobalClusterArn": "arn:aws:rds::111111111111:global-cluster:global-repl",
+ "GlobalClusterResourceId": "cluster-globalrepl",
+ "Status": "available",
+ })
+ clusters = {
+ value: cluster
+ for cluster in (writer, target)
+ for value in (cluster["DBClusterIdentifier"], cluster["DBClusterArn"])
+ }
+ _patch_mysql_replication_unit_topology(monkeypatch, m, topology)
+ monkeypatch.setattr(
+ m, "_resolve_global_cluster",
+ lambda identifier: global_cluster if identifier == "global-repl" else None,
+ )
+ monkeypatch.setattr(m, "_resolve_cluster", clusters.get)
+ monkeypatch.setattr(m, "_resolve_cluster_in_request_region", clusters.get)
+ monkeypatch.setattr(
+ m, "_cluster_member_instances",
+ lambda cluster: [{
+ "DBInstanceIdentifier": cluster["DBClusterMembers"][0]["DBInstanceIdentifier"],
+ "DBInstanceStatus": "available",
+ }] if cluster.get("DBClusterMembers") else [],
+ )
+ monkeypatch.setattr(m, "_mysql_global_writer_switch_owners", {})
+ return topology
+
+
+def _mysql_switchover_params(target="secondary"):
+ return {
+ "GlobalClusterIdentifier": "global-repl",
+ "TargetDbClusterIdentifier": target,
+ }
+
+
+def test_rds_mysql_gtid_wait_recomputes_shared_deadline(monkeypatch):
+ from ministack.services import rds as m
+
+ now = [100.0]
+ observed = {}
+ monkeypatch.setattr(m.time, "monotonic", lambda: now[0])
+
+ class Cursor:
+ def __init__(self, connection):
+ self.connection = connection
+
+ def execute(self, statement, params):
+ observed["sql"] = (statement, params)
+
+ def fetchone(self):
+ return (0,)
+
+ def close(self):
+ pass
+
+ class Connection:
+ def cursor(self):
+ now[0] += 4
+ return Cursor(self)
+
+ def close(self):
+ pass
+
+ def connect(_cluster, *, timeout):
+ observed["connect_timeout"] = timeout
+ return Connection()
+
+ monkeypatch.setattr(m, "_mysql_replication_connection", connect)
+
+ assert m._wait_for_mysql_gtid(
+ {"DBClusterIdentifier": "secondary"}, "source:1-9", 30,
+ deadline=130.0,
+ ) is True
+ assert observed["connect_timeout"] == 30.0
+ assert observed["sql"][1] == ("source:1-9", 26.0)
+
+
[email protected]("mutation", ["password", "topology"])
+def test_rds_active_switchover_rejects_mutation(monkeypatch, mutation):
+ from ministack.services import rds as m
+
+ writer, target, writer_member, target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+ with m._shared_container_lock:
+ owner = m._claim_mysql_global_writer_switch(global_cluster, writer, target)
+ if mutation == "password":
+ result = m._modify_db_cluster({
+ "DBClusterIdentifier": "primary",
+ "MasterUserPassword": "changed-password",
+ })
+ elif mutation == "topology":
+ result = m._remove_from_global_cluster({
+ "GlobalClusterIdentifier": "global-repl",
+ "DbClusterIdentifier": "secondary",
+ })
+ else:
+ result = m._failover_global_cluster({
+ "GlobalClusterIdentifier": "global-repl",
+ "TargetDbClusterIdentifier": "secondary",
+ "AllowDataLoss": "true",
+ })
+
+ assert result[0] == 400
+ assert m._active_mysql_global_writer_switch(global_cluster) is owner
+ assert writer_member["IsWriter"] is True
+ assert target_member["IsWriter"] is False
+ assert writer.get("_MasterUserPassword") != "changed-password"
+ assert target["GlobalClusterIdentifier"] == "global-repl"
+ assert target["_mysql_replication_source_arn"] == writer["DBClusterArn"]
+
+
+ ("handler_name", "impl_name"),
+ [
+ ("_create_db_instance", "_create_db_instance_impl"),
+ ("_delete_db_instance", "_delete_db_instance_impl"),
+ ],
+)
+def test_rds_instance_mutation_prevents_switchover_claim(
+ monkeypatch,
+ handler_name,
+ impl_name,
+):
+ from ministack.services import rds as m
+
+ writer, target, _writer_member, _target_member, global_cluster = (
+ _mysql_switchover_unit_topology(monkeypatch, m)
+ )
+ entered = threading.Event()
+ release = threading.Event()
+ claimed = threading.Event()
+ order = []
+
+ def mutation_impl(_params):
+ entered.set()
+ assert release.wait(timeout=1)
+ order.append("mutation")
+ return "mutated"
+
+ monkeypatch.setattr(m, impl_name, mutation_impl)
+ if handler_name == "_create_db_instance":
+ params = {"DBClusterIdentifier": "primary"}
+ else:
+ m._instances["mutating-instance"] = {
+ "DBInstanceIdentifier": "mutating-instance",
+ "DBClusterIdentifier": "primary",
+ }
+ params = {"DBInstanceIdentifier": "mutating-instance"}
+ mutation = threading.Thread(target=getattr(m, handler_name), args=(params,))
+ mutation.start()
+ assert entered.wait(timeout=1)
+
+ def claim():
+ with m._shared_container_lock:
+ owner = m._claim_mysql_global_writer_switch(
+ global_cluster, writer, target,
+ )
+ order.append(("claim", owner))
+ claimed.set()
+
+ claimant = threading.Thread(target=claim)
+ claimant.start()
+ assert claimed.wait(timeout=1)
+ assert order == [("claim", None)]
+ release.set()
+ mutation.join(timeout=1)
+ claimant.join(timeout=1)
+
+ assert not mutation.is_alive() and not claimant.is_alive()
+ assert order == [("claim", None), "mutation"]
+ with m._shared_container_lock:
+ assert m._claim_mysql_global_writer_switch(
+ global_cluster, writer, target,
+ ) is not None
+
+
+def test_rds_mysql_gtid_wait_and_write_enable_use_exact_sql(monkeypatch):
+ from ministack.services import rds as m
+
+ events = []
+ rows = iter([(0,), (0, 0)])
+
+ class Cursor:
+ def execute(self, statement, params=None):
+ events.append((statement, params))
+
+ def fetchone(self):
+ return next(rows)
+
+ def close(self):
+ events.append("cursor-close")
+
+ class Connection:
+ def cursor(self):
+ return Cursor()
+
+ def close(self):
+ events.append("connection-close")
+
+ monkeypatch.setattr(m, "_mysql_replication_connection", lambda *_a, **_k: Connection())
+ cluster = {"DBClusterIdentifier": "secondary"}
+ assert m._wait_for_mysql_gtid(cluster, "source:1-9", 10) is True
+ assert m._set_mysql_cluster_writable(cluster) is True
+ assert events == [
+ ("SELECT WAIT_FOR_EXECUTED_GTID_SET(%s, %s)", ("source:1-9", 10.0)),
+ "cursor-close", "connection-close",
+ ("SET GLOBAL super_read_only=OFF", None),
+ ("SET GLOBAL read_only=OFF", None),
+ ("SELECT @@GLOBAL.read_only, @@GLOBAL.super_read_only", None),
+ "cursor-close", "connection-close",
+ ]
+
+
+def test_rds_mysql_write_enable_refences_partial_failure(monkeypatch):
+ from ministack.services import rds as m
+
+ statements = []
+
+ class Cursor:
+ def execute(self, statement, _params=None):
+ statements.append(statement)
+ if statement == "SET GLOBAL read_only=OFF":
+ raise RuntimeError("read-only write failed")
+
+ def close(self):
+ pass
+
+ class Connection:
+ def cursor(self):
+ return Cursor()
+
+ def close(self):
+ pass
+
+ monkeypatch.setattr(m, "_mysql_replication_connection", lambda *_a, **_k: Connection())
+
+ assert m._set_mysql_cluster_writable({"DBClusterIdentifier": "target"}) is False
+ assert statements == [
+ "SET GLOBAL super_read_only=OFF",
+ "SET GLOBAL read_only=OFF",
+ "SET GLOBAL super_read_only=ON",
+ ]
+
+
+def test_rds_mysql_write_enable_refences_lost_disable_acknowledgment(monkeypatch):
+ from ministack.services import rds as m
+
+ statements = []
+ server_state = {"super_read_only": True}
+
+ class Cursor:
+ def execute(self, statement, _params=None):
+ statements.append(statement)
+ if statement == "SET GLOBAL super_read_only=OFF":
+ server_state["super_read_only"] = False
+ raise RuntimeError("write applied but acknowledgment was lost")
+ if statement == "SET GLOBAL super_read_only=ON":
+ server_state["super_read_only"] = True
+
+ def close(self):
+ pass
+
+ class Connection:
+ def cursor(self):
+ return Cursor()
+
+ def close(self):
+ pass
+
+ monkeypatch.setattr(m, "_mysql_replication_connection", lambda *_a, **_k: Connection())
+
+ assert m._set_mysql_cluster_writable({"DBClusterIdentifier": "target"}) is False
+ assert statements == [
+ "SET GLOBAL super_read_only=OFF",
+ "SET GLOBAL super_read_only=ON",
+ ]
+ assert server_state["super_read_only"] is True
+
+
[email protected]("close_failure", ["cursor", "connection"])
+def test_rds_mysql_verified_write_enable_survives_cleanup_failure(
+ monkeypatch,
+ close_failure,
+):
+ from ministack.services import rds as m
+
+ class Cursor:
+ def execute(self, _statement, _params=None):
+ pass
+
+ def fetchone(self):
+ return (0, 0)
+
+ def close(self):
+ if close_failure == "cursor":
+ raise RuntimeError("cursor close failed")
+
+ class Connection:
+ def cursor(self):
+ return Cursor()
+
+ def close(self):
+ if close_failure == "connection":
+ raise RuntimeError("connection close failed")
+
+ monkeypatch.setattr(m, "_mysql_replication_connection", lambda *_a, **_k: Connection())
+ assert m._set_mysql_cluster_writable({"DBClusterIdentifier": "target"}) is True
+
+
def _wait_for_replica_status(endpoint, timeout=120):
import pymysql