Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

feat(rds): MySQL instances apply their DB parameter group (#2030)

Jeff Hellera3f78c352432
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 747dc1a4..0710740c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).
### Fixed
+- **RDS — MySQL instances apply their DB parameter group** — a group's parameters were stored but never reached the server. A MySQL or MariaDB instance now starts with its group's set parameters as server options; `ModifyDBParameterGroup` with `ApplyMethod=immediate` applies dynamic parameters to running instances (`SET GLOBAL`), and `ResetDBParameterGroup` returns them to the engine default; an `immediate` change to a static parameter is refused with `InvalidParameterCombination`, as AWS refuses it, when a running instance uses the group (MiniStack has no parameter list, so with none it is stored); a `pending-reboot` change reports `ParameterApplyStatus: pending-reboot` until the instance next starts. Names and values are not validated against the engine's parameter list, and formula values (`{DBInstanceClassMemory*3/4}`) are not evaluated. Contributed by @skialpine.
- **API Gateway — `GetApiKeys` filters by `nameQuery`** — `GetApiKeys` ignored `nameQuery` and returned all the API keys, so a lookup by name could get the wrong key. It now returns only the keys whose names start with `nameQuery`.
- **RDS — creating an existing parameter group is refused** — `CreateDBParameterGroup` and `CreateDBClusterParameterGroup` with a name already in use replaced the group with an empty one, dropping its parameters; they now return `DBParameterGroupAlreadyExists`, as AWS does. Contributed by @skialpine.
- **RDS — a backup retention period of 0 turns binary logging off** — on RDS for MySQL, `BackupRetentionPeriod=0` turns binary logging off; MiniStack's MySQL 8.0/8.4 instances kept the image default, binary logging on. Such an instance now starts with binary logging off. Contributed by @skialpine.
diff --git a/ministack/services/rds.py b/ministack/services/rds.py
index f111e23d..b3790b4e 100644
--- a/ministack/services/rds.py
+++ b/ministack/services/rds.py
@@ -2457,10 +2457,13 @@ def _start_rds_container_for_instance(db_id, instance):
container_kwargs["tmpfs"] = {
data_path: f"rw,noexec,nosuid,size={RDS_TMPFS_SIZE}",
}
- if _is_mysql_engine(engine) and (
- options := _mysql_server_options(instance.get("BackupRetentionPeriod", 1))
- ):
+ groups = instance.get("DBParameterGroups") or [{}]
+ if _is_mysql_engine(engine) and (options := _mysql_server_options(
+ instance.get("BackupRetentionPeriod", 1), groups[0].get("DBParameterGroupName"),
+ )):
container_kwargs["command"] = options
+ if groups[0].get("ParameterApplyStatus") in ("pending-reboot", "applying"):
+ groups[0]["ParameterApplyStatus"] = "in-sync"
try:
container = _run_rds_container(
@@ -2623,10 +2626,95 @@ def _is_mysql_engine(engine):
return any(e in engine for e in ("mysql", "aurora-mysql", "mariadb"))
-def _mysql_server_options(backup_retention_period):
- """Server options for a standalone MySQL/MariaDB instance. As on RDS, a
- backup retention period of 0 turns binary logging off."""
- return [] if int(backup_retention_period) > 0 else ["--skip-log-bin"]
+# A DB parameter group name that is also a server variable (RDS-only names such
+# as `rds.force_ssl` are not).
+_MYSQL_SERVER_PARAMETER = re.compile(r"[A-Za-z0-9_]+")
+
+
+def _mysql_server_options(backup_retention_period, param_group_name=None):
+ """Server options for a standalone MySQL/MariaDB instance, as RDS starts one:
+ a backup retention period of 0 turns binary logging off, and the instance's
+ DB parameter group's set values are server options. `--loose-` keeps a name
+ the server does not know as a startup option (such as `time_zone`) from
+ stopping it; formula values (`{DBInstanceClassMemory*3/4}`) are not evaluated."""
+ options = [] if int(backup_retention_period) > 0 else ["--skip-log-bin"]
+ group = _param_groups.get(param_group_name) if param_group_name else None
+ for name, param in ((group or {}).get("Parameters") or {}).items():
+ value = param.get("ParameterValue")
+ if _MYSQL_SERVER_PARAMETER.fullmatch(name) and value is not None and not value.startswith("{"):
+ options.append(f"--loose-{name}={value}")
+ return options
+
+
+def _mysql_parameter_value(value):
+ """A parameter value as SET GLOBAL takes it: numeric variables refuse a string."""
+ for parse in (int, float):
+ try:
+ return parse(value)
+ except ValueError:
+ pass
+ return value
+
+
+def _apply_parameter_group_changes(group_name, changes, refuse_static=True):
+ """Apply `(name, value, apply_method)` changes to the running MySQL instances
+ in a DB parameter group, as RDS does: an `immediate` change to a dynamic
+ parameter takes effect now (a `None` value resets it to the engine default);
+ a `pending-reboot` change leaves the instance `pending-reboot` until the next
+ start, which applies the whole group (`_mysql_server_options`). An `immediate`
+ change to a static parameter, which the server reports as a read-only
+ variable, is refused as AWS refuses it, before anything is applied; with
+ `refuse_static=False` (a reset) it is left pending-reboot instead. Returns
+ the refusal, or None."""
+ for instance in list(_instances.values()):
+ groups = instance.get("DBParameterGroups") or []
+ if not (
+ groups and groups[0].get("DBParameterGroupName") == group_name
+ and _is_mysql_engine(instance.get("Engine", ""))
+ and instance.get("_docker_container_id")
+ and not instance.get("DBClusterIdentifier")
+ ):
+ continue
+ pending = any(method != "immediate" for _name, _value, method in changes)
+ immediate = [(name, value) for name, value, method in changes
+ if method == "immediate" and _MYSQL_SERVER_PARAMETER.fullmatch(name)]
+ if immediate:
+ try:
+ conn = _mysql_endpoint_admin_connection(
+ instance.get("_internal_address") or "127.0.0.1",
+ instance.get("_internal_port") or instance.get("_HostPort"),
+ instance.get("_MasterUserPassword", "password"),
+ )
+ except Exception as e:
+ logger.warning("RDS: cannot apply parameter group %s to %s: %s",
+ group_name, instance.get("DBInstanceIdentifier"), e)
+ pending = True
+ else:
+ try:
+ with conn.cursor() as cur:
+ for name, _value in immediate if refuse_static else ():
+ try:
+ cur.execute(f"SET GLOBAL {name} = @@GLOBAL.{name}")
+ except Exception as e:
+ if e.args and e.args[0] == 1238: # ER_INCORRECT_GLOBAL_LOCAL_VAR: read-only
+ return _error("InvalidParameterCombination",
+ "cannot use immediate apply method for static parameter", 400)
+ for name, value in immediate:
+ try:
+ if value is None:
+ cur.execute(f"SET GLOBAL {name} = DEFAULT")
+ else:
+ cur.execute(f"SET GLOBAL {name} = %s", (_mysql_parameter_value(value),))
+ except Exception as e:
+ # A static (read-only) variable takes effect at the next start.
+ logger.info("RDS: parameter %s for %s applies at the next start: %s",
+ name, instance.get("DBInstanceIdentifier"), e)
+ pending = True
+ finally:
+ conn.close()
+ if pending:
+ groups[0]["ParameterApplyStatus"] = "pending-reboot"
+ return None
def _is_postgres_engine(engine):
@@ -4785,9 +4873,9 @@ def _create_db_instance_impl(p):
container_kwargs["tmpfs"] = {
data_path: f"rw,noexec,nosuid,size={RDS_TMPFS_SIZE}",
}
- if _is_mysql_engine(engine) and (
- options := _mysql_server_options(_p(p, "BackupRetentionPeriod") or "1")
- ):
+ if _is_mysql_engine(engine) and (options := _mysql_server_options(
+ _p(p, "BackupRetentionPeriod") or "1", param_group_name,
+ )):
container_kwargs["command"] = options
container = _run_rds_container(
docker_client, engine, container_kwargs,
@@ -6546,13 +6634,19 @@ def _modify_param_group(p):
params = pg.setdefault("Parameters", {})
prefix = _parameter_member_prefix(p)
+ changes = []
idx = 1
while _p(p, f"{prefix}.{idx}.ParameterName"):
pname = _p(p, f"{prefix}.{idx}.ParameterName")
pvalue = _p(p, f"{prefix}.{idx}.ParameterValue")
apply_method = _p(p, f"{prefix}.{idx}.ApplyMethod") or "immediate"
- params[pname] = {"ParameterValue": pvalue, "ApplyMethod": apply_method}
+ changes.append((pname, pvalue, apply_method))
idx += 1
+ refusal = _apply_parameter_group_changes(name, changes)
+ if refusal:
+ return refusal
+ for pname, pvalue, apply_method in changes:
+ params[pname] = {"ParameterValue": pvalue, "ApplyMethod": apply_method}
return _xml(200, "ModifyDBParameterGroupResponse",
f"<ModifyDBParameterGroupResult><DBParameterGroupName>{name}</DBParameterGroupName></ModifyDBParameterGroupResult>")
@@ -6576,12 +6670,17 @@ def _reset_param_group(p):
)
if reset_all or not has_explicit_parameters:
+ reset = [(pname, None, "immediate") for pname in params]
params.clear()
else:
+ reset = []
idx = 1
while _p(p, f"{prefix}.{idx}.ParameterName"):
- params.pop(_p(p, f"{prefix}.{idx}.ParameterName"), None)
+ pname = _p(p, f"{prefix}.{idx}.ParameterName")
+ if params.pop(pname, None) is not None:
+ reset.append((pname, None, _p(p, f"{prefix}.{idx}.ApplyMethod") or "immediate"))
idx += 1
+ _apply_parameter_group_changes(name, reset, refuse_static=False)
return _xml(200, "ResetDBParameterGroupResponse",
f"<ResetDBParameterGroupResult><DBParameterGroupName>{name}</DBParameterGroupName></ResetDBParameterGroupResult>")
diff --git a/tests/test_rds.py b/tests/test_rds.py
index 566f2dec..67313d9f 100644
--- a/tests/test_rds.py
+++ b/tests/test_rds.py
@@ -8234,6 +8234,60 @@ def test_rds_mysql_binary_logging_follows_backup_retention(rds):
rds.delete_db_instance(DBInstanceIdentifier=db_id, SkipFinalSnapshot=True)
+def test_rds_mysql_applies_db_parameter_group(rds):
+ """An instance starts with its DB parameter group (a name mysqld only takes
+ under another option, `time_zone`, does not stop it). An immediate change to
+ a dynamic parameter applies at once and a reset returns it to the default; AWS
+ refuses an immediate change to a static one, which pending-reboot accepts."""
+ suffix = uuid.uuid4().hex[:8]
+ group, db_id = f"pg-{suffix}", f"pg-db-{suffix}"
+ rds.create_db_parameter_group(DBParameterGroupName=group,
+ DBParameterGroupFamily="mysql8.0", Description="test")
+
+ def modify(name, value, method="immediate"):
+ rds.modify_db_parameter_group(DBParameterGroupName=group, Parameters=[
+ {"ParameterName": name, "ParameterValue": value, "ApplyMethod": method}])
+
+ def status():
+ instance = rds.describe_db_instances(DBInstanceIdentifier=db_id)["DBInstances"][0]
+ return instance["DBParameterGroups"][0]["ParameterApplyStatus"]
+
+ modify("collation_server", "utf8mb4_bin")
+ modify("time_zone", "UTC")
+ try:
+ rds.create_db_instance(
+ DBInstanceIdentifier=db_id, Engine="mysql", EngineVersion="8.0",
+ DBInstanceClass="db.t3.micro", AllocatedStorage=20, DBName=DATABASE,
+ MasterUsername="admin", MasterUserPassword=PASSWORD, DBParameterGroupName=group,
+ )
+ conn = _aurora_connect(_wait_for_instance(rds, db_id)["Endpoint"])
+ cur = conn.cursor()
+
+ def variable(name):
+ cur.execute(f"SELECT @@GLOBAL.{name}")
+ return cur.fetchone()[0]
+
+ assert variable("collation_server") == "utf8mb4_bin"
+ modify("max_connections", "300")
+ assert (variable("max_connections"), status()) == (300, "in-sync")
+ modify("long_query_time", "0.5")
+ assert (float(variable("long_query_time")), status()) == (0.5, "in-sync")
+ rds.reset_db_parameter_group(DBParameterGroupName=group, Parameters=[
+ {"ParameterName": "max_connections", "ApplyMethod": "immediate"}])
+ assert variable("max_connections") == 151
+ with pytest.raises(ClientError) as exc:
+ modify("performance_schema", "0")
+ assert exc.value.response["Error"]["Code"] == "InvalidParameterCombination"
+ assert status() == "in-sync"
+ modify("performance_schema", "0", "pending-reboot")
+ assert status() == "pending-reboot"
+ conn.close()
+ finally:
+ rds.delete_db_instance(DBInstanceIdentifier=db_id, SkipFinalSnapshot=True)
+ rds.delete_db_parameter_group(DBParameterGroupName=group)
+
+
@contextlib.contextmanager
def _live_cluster(rds, engine_version=None):
suffix = uuid.uuid4().hex[:10]