Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
feat(rds): MySQL instances apply their DB parameter group (#2030)
Jeff Hellera3f78c352432
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 747dc1a4..0710740c 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).### Fixed+- **RDS — MySQL instances apply their DB parameter group** — a group's parameters were stored but never reached the server. A MySQL or MariaDB instance now starts with its group's set parameters as server options; `ModifyDBParameterGroup` with `ApplyMethod=immediate` applies dynamic parameters to running instances (`SET GLOBAL`), and `ResetDBParameterGroup` returns them to the engine default; an `immediate` change to a static parameter is refused with `InvalidParameterCombination`, as AWS refuses it, when a running instance uses the group (MiniStack has no parameter list, so with none it is stored); a `pending-reboot` change reports `ParameterApplyStatus: pending-reboot` until the instance next starts. Names and values are not validated against the engine's parameter list, and formula values (`{DBInstanceClassMemory*3/4}`) are not evaluated. Contributed by @skialpine.- **API Gateway — `GetApiKeys` filters by `nameQuery`** — `GetApiKeys` ignored `nameQuery` and returned all the API keys, so a lookup by name could get the wrong key. It now returns only the keys whose names start with `nameQuery`.- **RDS — creating an existing parameter group is refused** — `CreateDBParameterGroup` and `CreateDBClusterParameterGroup` with a name already in use replaced the group with an empty one, dropping its parameters; they now return `DBParameterGroupAlreadyExists`, as AWS does. Contributed by @skialpine.- **RDS — a backup retention period of 0 turns binary logging off** — on RDS for MySQL, `BackupRetentionPeriod=0` turns binary logging off; MiniStack's MySQL 8.0/8.4 instances kept the image default, binary logging on. Such an instance now starts with binary logging off. Contributed by @skialpine.diff --git a/ministack/services/rds.py b/ministack/services/rds.pyindex f111e23d..b3790b4e 100644--- a/ministack/services/rds.py+++ b/ministack/services/rds.py@@ -2457,10 +2457,13 @@ def _start_rds_container_for_instance(db_id, instance):container_kwargs["tmpfs"] = {data_path: f"rw,noexec,nosuid,size={RDS_TMPFS_SIZE}",}- if _is_mysql_engine(engine) and (- options := _mysql_server_options(instance.get("BackupRetentionPeriod", 1))- ):+ groups = instance.get("DBParameterGroups") or [{}]+ if _is_mysql_engine(engine) and (options := _mysql_server_options(+ instance.get("BackupRetentionPeriod", 1), groups[0].get("DBParameterGroupName"),+ )):container_kwargs["command"] = options+ if groups[0].get("ParameterApplyStatus") in ("pending-reboot", "applying"):+ groups[0]["ParameterApplyStatus"] = "in-sync"try:container = _run_rds_container(@@ -2623,10 +2626,95 @@ def _is_mysql_engine(engine):return any(e in engine for e in ("mysql", "aurora-mysql", "mariadb"))-def _mysql_server_options(backup_retention_period):- """Server options for a standalone MySQL/MariaDB instance. As on RDS, a- backup retention period of 0 turns binary logging off."""- return [] if int(backup_retention_period) > 0 else ["--skip-log-bin"]+# A DB parameter group name that is also a server variable (RDS-only names such+# as `rds.force_ssl` are not).+_MYSQL_SERVER_PARAMETER = re.compile(r"[A-Za-z0-9_]+")+++def _mysql_server_options(backup_retention_period, param_group_name=None):+ """Server options for a standalone MySQL/MariaDB instance, as RDS starts one:+ a backup retention period of 0 turns binary logging off, and the instance's+ DB parameter group's set values are server options. `--loose-` keeps a name+ the server does not know as a startup option (such as `time_zone`) from+ stopping it; formula values (`{DBInstanceClassMemory*3/4}`) are not evaluated."""+ options = [] if int(backup_retention_period) > 0 else ["--skip-log-bin"]+ group = _param_groups.get(param_group_name) if param_group_name else None+ for name, param in ((group or {}).get("Parameters") or {}).items():+ value = param.get("ParameterValue")+ if _MYSQL_SERVER_PARAMETER.fullmatch(name) and value is not None and not value.startswith("{"):+ options.append(f"--loose-{name}={value}")+ return options+++def _mysql_parameter_value(value):+ """A parameter value as SET GLOBAL takes it: numeric variables refuse a string."""+ for parse in (int, float):+ try:+ return parse(value)+ except ValueError:+ pass+ return value+++def _apply_parameter_group_changes(group_name, changes, refuse_static=True):+ """Apply `(name, value, apply_method)` changes to the running MySQL instances+ in a DB parameter group, as RDS does: an `immediate` change to a dynamic+ parameter takes effect now (a `None` value resets it to the engine default);+ a `pending-reboot` change leaves the instance `pending-reboot` until the next+ start, which applies the whole group (`_mysql_server_options`). An `immediate`+ change to a static parameter, which the server reports as a read-only+ variable, is refused as AWS refuses it, before anything is applied; with+ `refuse_static=False` (a reset) it is left pending-reboot instead. Returns+ the refusal, or None."""+ for instance in list(_instances.values()):+ groups = instance.get("DBParameterGroups") or []+ if not (+ groups and groups[0].get("DBParameterGroupName") == group_name+ and _is_mysql_engine(instance.get("Engine", ""))+ and instance.get("_docker_container_id")+ and not instance.get("DBClusterIdentifier")+ ):+ continue+ pending = any(method != "immediate" for _name, _value, method in changes)+ immediate = [(name, value) for name, value, method in changes+ if method == "immediate" and _MYSQL_SERVER_PARAMETER.fullmatch(name)]+ if immediate:+ try:+ conn = _mysql_endpoint_admin_connection(+ instance.get("_internal_address") or "127.0.0.1",+ instance.get("_internal_port") or instance.get("_HostPort"),+ instance.get("_MasterUserPassword", "password"),+ )+ except Exception as e:+ logger.warning("RDS: cannot apply parameter group %s to %s: %s",+ group_name, instance.get("DBInstanceIdentifier"), e)+ pending = True+ else:+ try:+ with conn.cursor() as cur:+ for name, _value in immediate if refuse_static else ():+ try:+ cur.execute(f"SET GLOBAL {name} = @@GLOBAL.{name}")+ except Exception as e:+ if e.args and e.args[0] == 1238: # ER_INCORRECT_GLOBAL_LOCAL_VAR: read-only+ return _error("InvalidParameterCombination",+ "cannot use immediate apply method for static parameter", 400)+ for name, value in immediate:+ try:+ if value is None:+ cur.execute(f"SET GLOBAL {name} = DEFAULT")+ else:+ cur.execute(f"SET GLOBAL {name} = %s", (_mysql_parameter_value(value),))+ except Exception as e:+ # A static (read-only) variable takes effect at the next start.+ logger.info("RDS: parameter %s for %s applies at the next start: %s",+ name, instance.get("DBInstanceIdentifier"), e)+ pending = True+ finally:+ conn.close()+ if pending:+ groups[0]["ParameterApplyStatus"] = "pending-reboot"+ return Nonedef _is_postgres_engine(engine):@@ -4785,9 +4873,9 @@ def _create_db_instance_impl(p):container_kwargs["tmpfs"] = {data_path: f"rw,noexec,nosuid,size={RDS_TMPFS_SIZE}",}- if _is_mysql_engine(engine) and (- options := _mysql_server_options(_p(p, "BackupRetentionPeriod") or "1")- ):+ if _is_mysql_engine(engine) and (options := _mysql_server_options(+ _p(p, "BackupRetentionPeriod") or "1", param_group_name,+ )):container_kwargs["command"] = optionscontainer = _run_rds_container(docker_client, engine, container_kwargs,@@ -6546,13 +6634,19 @@ def _modify_param_group(p):params = pg.setdefault("Parameters", {})prefix = _parameter_member_prefix(p)+ changes = []idx = 1while _p(p, f"{prefix}.{idx}.ParameterName"):pname = _p(p, f"{prefix}.{idx}.ParameterName")pvalue = _p(p, f"{prefix}.{idx}.ParameterValue")apply_method = _p(p, f"{prefix}.{idx}.ApplyMethod") or "immediate"- params[pname] = {"ParameterValue": pvalue, "ApplyMethod": apply_method}+ changes.append((pname, pvalue, apply_method))idx += 1+ refusal = _apply_parameter_group_changes(name, changes)+ if refusal:+ return refusal+ for pname, pvalue, apply_method in changes:+ params[pname] = {"ParameterValue": pvalue, "ApplyMethod": apply_method}return _xml(200, "ModifyDBParameterGroupResponse",f"<ModifyDBParameterGroupResult><DBParameterGroupName>{name}</DBParameterGroupName></ModifyDBParameterGroupResult>")@@ -6576,12 +6670,17 @@ def _reset_param_group(p):)if reset_all or not has_explicit_parameters:+ reset = [(pname, None, "immediate") for pname in params]params.clear()else:+ reset = []idx = 1while _p(p, f"{prefix}.{idx}.ParameterName"):- params.pop(_p(p, f"{prefix}.{idx}.ParameterName"), None)+ pname = _p(p, f"{prefix}.{idx}.ParameterName")+ if params.pop(pname, None) is not None:+ reset.append((pname, None, _p(p, f"{prefix}.{idx}.ApplyMethod") or "immediate"))idx += 1+ _apply_parameter_group_changes(name, reset, refuse_static=False)return _xml(200, "ResetDBParameterGroupResponse",f"<ResetDBParameterGroupResult><DBParameterGroupName>{name}</DBParameterGroupName></ResetDBParameterGroupResult>")diff --git a/tests/test_rds.py b/tests/test_rds.pyindex 566f2dec..67313d9f 100644--- a/tests/test_rds.py+++ b/tests/test_rds.py@@ -8234,6 +8234,60 @@ def test_rds_mysql_binary_logging_follows_backup_retention(rds):rds.delete_db_instance(DBInstanceIdentifier=db_id, SkipFinalSnapshot=True)[email protected]_plane+def test_rds_mysql_applies_db_parameter_group(rds):+ """An instance starts with its DB parameter group (a name mysqld only takes+ under another option, `time_zone`, does not stop it). An immediate change to+ a dynamic parameter applies at once and a reset returns it to the default; AWS+ refuses an immediate change to a static one, which pending-reboot accepts."""+ suffix = uuid.uuid4().hex[:8]+ group, db_id = f"pg-{suffix}", f"pg-db-{suffix}"+ rds.create_db_parameter_group(DBParameterGroupName=group,+ DBParameterGroupFamily="mysql8.0", Description="test")++ def modify(name, value, method="immediate"):+ rds.modify_db_parameter_group(DBParameterGroupName=group, Parameters=[+ {"ParameterName": name, "ParameterValue": value, "ApplyMethod": method}])++ def status():+ instance = rds.describe_db_instances(DBInstanceIdentifier=db_id)["DBInstances"][0]+ return instance["DBParameterGroups"][0]["ParameterApplyStatus"]++ modify("collation_server", "utf8mb4_bin")+ modify("time_zone", "UTC")+ try:+ rds.create_db_instance(+ DBInstanceIdentifier=db_id, Engine="mysql", EngineVersion="8.0",+ DBInstanceClass="db.t3.micro", AllocatedStorage=20, DBName=DATABASE,+ MasterUsername="admin", MasterUserPassword=PASSWORD, DBParameterGroupName=group,+ )+ conn = _aurora_connect(_wait_for_instance(rds, db_id)["Endpoint"])+ cur = conn.cursor()++ def variable(name):+ cur.execute(f"SELECT @@GLOBAL.{name}")+ return cur.fetchone()[0]++ assert variable("collation_server") == "utf8mb4_bin"+ modify("max_connections", "300")+ assert (variable("max_connections"), status()) == (300, "in-sync")+ modify("long_query_time", "0.5")+ assert (float(variable("long_query_time")), status()) == (0.5, "in-sync")+ rds.reset_db_parameter_group(DBParameterGroupName=group, Parameters=[+ {"ParameterName": "max_connections", "ApplyMethod": "immediate"}])+ assert variable("max_connections") == 151+ with pytest.raises(ClientError) as exc:+ modify("performance_schema", "0")+ assert exc.value.response["Error"]["Code"] == "InvalidParameterCombination"+ assert status() == "in-sync"+ modify("performance_schema", "0", "pending-reboot")+ assert status() == "pending-reboot"+ conn.close()+ finally:+ rds.delete_db_instance(DBInstanceIdentifier=db_id, SkipFinalSnapshot=True)+ rds.delete_db_parameter_group(DBParameterGroupName=group)++@contextlib.contextmanagerdef _live_cluster(rds, engine_version=None):suffix = uuid.uuid4().hex[:10]