Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

feat(cloudformation): update S3 Tables table buckets and tables in place (#1886)

iot-rocket6e77355e22f4
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e4c19406..5d518141 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).
### Fixed
- **Cognito — `USER_SRP_AUTH` checks the password** — `PASSWORD_VERIFIER` answered random `SALT` and `SRP_B` values and accepted any response, so a wrong password, a missing `TIMESTAMP` or a disabled or unconfirmed user all got tokens. The challenge now carries SRP-6a parameters with a stable salt per user, and the response is refused with `NotAuthorizedException` unless its signature proves the stored password. `TIMESTAMP` must read `EEE MMM d HH:mm:ss z yyyy` with `UTC` or `GMT` as the zone, the signature is computed over its canonical form, `PASSWORD_CLAIM_SIGNATURE` must be base64, and a temporary password leads to `NEW_PASSWORD_REQUIRED`. A client without `ALLOW_USER_SRP_AUTH`, an unknown user under `LEGACY`, a disabled and an unconfirmed user are refused at `InitiateAuth`. The built-in `PASSWORD_VERIFIER` step of `CUSTOM_WITH_SRP` gets the same check. Contributed by @iot-rocket.
- **Cognito — refresh tokens are checked against the pool and client** — `REFRESH_TOKEN_AUTH` (`InitiateAuth`, `AdminInitiateAuth`) and `GetTokensFromRefreshToken` issued tokens for the first user in the pool when the refresh token was malformed, from another pool or of a deleted user, and accepted a token issued to another client; they now answer `NotAuthorizedException` with the message AWS gives for each case, and the refresh token from a SAML or OIDC federated sign-in now refreshes the federated user's session. Contributed by @iot-rocket.
+- **CloudFormation — `AWS::S3Tables::TableBucket` and `AWS::S3Tables::Table` update in place** — a stack update re-ran the create, which reset the bucket's `createdAt` and rebuilt the table with a new `createdAt` and its initial metadata location, dropping committed metadata; a maintenance setting or tag change now keeps both, and a create-only table change under an unchanged name fails with the conflict AWS reports. Contributed by @iot-rocket.
- **RDS — `DescribeGlobalClusters` lists clusters as `GlobalClusterMember`** — each cluster was a `<GlobalCluster>` element, so SDKs that match the list member name, such as aws-sdk-go-v2, found none. Contributed by @IamYipi.
- **Amazon MQ — `DescribeBrokerInstanceOptions` returns `supportedEngineVersions` as strings** — they were `{"name": ...}` objects, which aws-sdk-go-v2 could not deserialize. Contributed by @IamYipi.
diff --git a/ministack/services/cloudformation/provisioners.py b/ministack/services/cloudformation/provisioners.py
index 46055e3a..2f0bbde0 100644
--- a/ministack/services/cloudformation/provisioners.py
+++ b/ministack/services/cloudformation/provisioners.py
@@ -10292,8 +10292,12 @@ def _backup_plan_update(physical_id, old_props, new_props, stack_name,
}
+def _s3tables_bucket_name(props, stack_name, logical_id):
+ return props.get("TableBucketName") or _physical_name(stack_name, logical_id, lowercase=True, max_len=63)
+
+
def _s3tables_bucket_create(logical_id, props, stack_name):
- name = props.get("TableBucketName") or _physical_name(stack_name, logical_id, lowercase=True, max_len=63)
+ name = _s3tables_bucket_name(props, stack_name, logical_id)
arn = _s3tables._bucket_arn(name)
_s3tables._table_buckets[name] = {
"arn": arn, "name": name,
@@ -10304,6 +10308,20 @@ def _s3tables_bucket_create(logical_id, props, stack_name):
return arn, {"TableBucketARN": arn}
+def _s3tables_bucket_update(physical_id, old_props, new_props, stack_name, logical_id=None):
+ """TableBucketName replaces the bucket; the other properties keep it."""
+ name = physical_id.rsplit("/", 1)[-1]
+ replaced = _rename_replacement(
+ physical_id, old_props, new_props, stack_name, logical_id,
+ _s3tables_bucket_name(new_props, stack_name, logical_id or physical_id),
+ name if name in _s3tables._table_buckets else None,
+ _s3tables_bucket_create, _s3tables_bucket_delete,
+ )
+ if replaced is not None:
+ return replaced
+ return physical_id, {"TableBucketARN": physical_id}
+
+
def _s3tables_bucket_delete(physical_id, props):
name = physical_id.rsplit("/", 1)[-1]
_s3tables._table_buckets.pop(name, None)
@@ -10328,10 +10346,30 @@ def _s3tables_namespace_delete(physical_id, props):
_s3tables._namespaces.pop(_s3tables._ns_key(bucket_arn, namespace), None)
+_S3TABLES_TABLE_CREATE_ONLY = (
+ "TableBucketARN", "OpenTableFormat", "IcebergMetadata", "WithoutMetadata",
+ "StorageClassConfiguration",
+)
+
+
+def _s3tables_table_key(props):
+ return _s3tables._table_key(
+ props.get("TableBucketARN", ""), props.get("Namespace", ""), props.get("TableName", ""))
+
+
+def _s3tables_table_attrs(table):
+ return {"TableARN": table["tableARN"], "TableBucketARN": table["tableBucketARN"],
+ "WarehouseLocation": table["warehouseLocation"], "Namespace": table["namespace"][0],
+ "TableName": table["name"]}
+
+
def _s3tables_table_create(logical_id, props, stack_name):
bucket_arn = props.get("TableBucketARN", "")
namespace = props.get("Namespace", "")
table_name = props.get("TableName", "")
+ key = _s3tables_table_key(props)
+ if key in _s3tables._tables:
+ raise ValueError("A table with an identical name already exists in the namespace.")
bucket_name = bucket_arn.rsplit("/", 1)[-1]
location = f"s3://{bucket_name}/{namespace}/{table_name}"
# IcebergMetadata.IcebergSchema.SchemaFieldList is how CFN (and CDK's
@@ -10350,7 +10388,6 @@ def _s3tables_table_create(logical_id, props, stack_name):
iceberg_metadata = _s3tables._initial_iceberg_metadata(table_name, schema_fields, location)
metadata_location = f"s3://{bucket_name}/{namespace}/{table_name}/metadata/v0.metadata.json"
table_arn = _s3tables._table_arn(bucket_arn, namespace, table_name)
- key = _s3tables._table_key(bucket_arn, namespace, table_name)
_s3tables._tables[key] = {
"name": table_name, "tableARN": table_arn, "namespace": [namespace],
"tableBucketARN": bucket_arn, "format": "ICEBERG",
@@ -10364,16 +10401,28 @@ def _s3tables_table_create(logical_id, props, stack_name):
if b["arn"] == bucket_arn:
b["tableCount"] = b.get("tableCount", 0) + 1
break
- return table_arn, {"TableARN": table_arn, "TableBucketARN": bucket_arn,
- "WarehouseLocation": location, "Namespace": namespace,
- "TableName": table_name}
+ return table_arn, _s3tables_table_attrs(_s3tables._tables[key])
+
+
+def _s3tables_table_update(physical_id, old_props, new_props, stack_name):
+ """Compaction, SnapshotManagement and Tags keep the table; any other change replaces it."""
+ old_key = _s3tables_table_key(old_props)
+ table = _s3tables._tables.get(old_key)
+ replaced = _rename_replacement(
+ physical_id, old_props, new_props, stack_name, None,
+ _s3tables_table_key(new_props), old_key if table else None,
+ _s3tables_table_create, _s3tables_table_delete,
+ )
+ if replaced is not None:
+ return replaced
+ if any(old_props.get(p) != new_props.get(p) for p in _S3TABLES_TABLE_CREATE_ONLY):
+ # The replacement is created first, so its unchanged name is refused.
+ return _s3tables_table_create(physical_id, new_props, stack_name)
+ return physical_id, _s3tables_table_attrs(table)
def _s3tables_table_delete(physical_id, props):
- bucket_arn = props.get("TableBucketARN", "")
- namespace = props.get("Namespace", "")
- table_name = props.get("TableName", "")
- _s3tables._tables.pop(_s3tables._table_key(bucket_arn, namespace, table_name), None)
+ _s3tables._tables.pop(_s3tables_table_key(props), None)
# --- Kinesis Data Firehose DeliveryStream ---
@@ -11271,9 +11320,18 @@ _RESOURCE_HANDLERS = {
"update": _s3_bucket_policy_update,
"delete": _s3_bucket_policy_delete,
},
- "AWS::S3Tables::TableBucket": {"create": _s3tables_bucket_create, "delete": _s3tables_bucket_delete},
+ "AWS::S3Tables::TableBucket": {
+ "create": _s3tables_bucket_create,
+ "update": _s3tables_bucket_update,
+ "update_with_logical_id": True,
+ "delete": _s3tables_bucket_delete,
+ },
"AWS::S3Tables::Namespace": {"create": _s3tables_namespace_create, "delete": _s3tables_namespace_delete},
- "AWS::S3Tables::Table": {"create": _s3tables_table_create, "delete": _s3tables_table_delete},
+ "AWS::S3Tables::Table": {
+ "create": _s3tables_table_create,
+ "update": _s3tables_table_update,
+ "delete": _s3tables_table_delete,
+ },
"AWS::SQS::Queue": {
"create": _sqs_create,
"update": _sqs_update,
diff --git a/tests/test_cfn.py b/tests/test_cfn.py
index 2c098599..f5ee5c9c 100644
--- a/tests/test_cfn.py
+++ b/tests/test_cfn.py
@@ -15733,6 +15733,118 @@ def test_cfn_s3tables_table_schema_from_iceberg_metadata(cfn, s3tables):
_wait_stack(cfn, stack_name)
+def _cfn_s3tables_template(bucket, days=4, target_mb=512, min_snapshots=1, table="t1",
+ columns=("id",)):
+ return json.dumps({"Resources": {
+ "Bucket": {"Type": "AWS::S3Tables::TableBucket", "Properties": {
+ "TableBucketName": bucket,
+ "UnreferencedFileRemoval": {"Status": "Enabled", "UnreferencedDays": days}}},
+ "Ns": {"Type": "AWS::S3Tables::Namespace", "Properties": {
+ "TableBucketARN": {"Ref": "Bucket"}, "Namespace": "ns1"}},
+ "Table": {"Type": "AWS::S3Tables::Table", "DependsOn": "Ns", "Properties": {
+ "TableBucketARN": {"Ref": "Bucket"}, "Namespace": "ns1", "TableName": table,
+ "OpenTableFormat": "ICEBERG",
+ "IcebergMetadata": {"IcebergSchema": {"SchemaFieldList": [
+ {"Name": c, "Type": "int"} for c in columns]}},
+ "Compaction": {"Status": "enabled", "TargetFileSizeMB": target_mb},
+ "SnapshotManagement": {"Status": "enabled", "MinSnapshotsToKeep": min_snapshots}}},
+ }, "Outputs": {"BucketArn": {"Value": {"Ref": "Bucket"}},
+ "TableArn": {"Value": {"Ref": "Table"}}}})
+
+
+def _cfn_s3tables_committed_stack(cfn, s3tables, stack_name, bucket):
+ """A created stack whose table has a committed metadata location; returns the bucket ARN."""
+ cfn.create_stack(StackName=stack_name, TemplateBody=_cfn_s3tables_template(bucket))
+ stack = _wait_stack(cfn, stack_name)
+ assert stack["StackStatus"] == "CREATE_COMPLETE", stack.get("StackStatusReason")
+ bucket_arn = _cfn_output(cfn, stack_name, "BucketArn")
+ current = s3tables.get_table_metadata_location(
+ tableBucketARN=bucket_arn, namespace="ns1", name="t1")
+ s3tables.update_table_metadata_location(
+ tableBucketARN=bucket_arn, namespace="ns1", name="t1",
+ versionToken=current["versionToken"],
+ metadataLocation=current["metadataLocation"].replace("v0", "v1"))
+ return bucket_arn
+
+
+def _cfn_s3tables_identity(s3tables, bucket_arn, table="t1"):
+ t = s3tables.get_table(tableBucketARN=bucket_arn, namespace="ns1", name=table)
+ return (s3tables.get_table_bucket(tableBucketARN=bucket_arn)["createdAt"],
+ t["tableARN"], t["createdAt"], t["metadataLocation"])
+
+
[email protected]("change", [
+ {"days": 5}, {"target_mb": 256}, {"min_snapshots": 2}])
+def test_cfn_s3tables_update_keeps_the_bucket_and_table(cfn, s3tables, change):
+ """A mutable property change keeps the bucket and the table with its committed metadata."""
+ suffix = _uuid_mod.uuid4().hex[:8]
+ stack_name, bucket = f"cfn-s3t-upd-{suffix}", f"cfn-s3t-upd-{suffix}"
+ try:
+ bucket_arn = _cfn_s3tables_committed_stack(cfn, s3tables, stack_name, bucket)
+ before = _cfn_s3tables_identity(s3tables, bucket_arn)
+ assert before[3].endswith("/v1.metadata.json")
+
+ cfn.update_stack(StackName=stack_name,
+ TemplateBody=_cfn_s3tables_template(bucket, **change))
+ stack = _wait_stack(cfn, stack_name)
+ assert stack["StackStatus"] == "UPDATE_COMPLETE", stack.get("StackStatusReason")
+ assert _cfn_s3tables_identity(s3tables, bucket_arn) == before
+ assert _cfn_output(cfn, stack_name, "TableArn") == before[1]
+ finally:
+ _delete_cfn_test_stack(cfn, stack_name)
+
+
+def test_cfn_s3tables_update_is_rolled_back(cfn, s3tables):
+ """A rolled-back bucket and table change keeps both as they were."""
+ suffix = _uuid_mod.uuid4().hex[:8]
+ stack_name, bucket = f"cfn-s3t-rb-{suffix}", f"cfn-s3t-rb-{suffix}"
+ try:
+ bucket_arn = _cfn_s3tables_committed_stack(cfn, s3tables, stack_name, bucket)
+ before = _cfn_s3tables_identity(s3tables, bucket_arn)
+
+ cfn.update_stack(StackName=stack_name, TemplateBody=_cfn_with_failing_resource(
+ _cfn_s3tables_template(bucket, days=6, target_mb=128), "Table"))
+ stack = _wait_stack(cfn, stack_name)
+ assert stack["StackStatus"] == "UPDATE_ROLLBACK_COMPLETE", stack.get("StackStatusReason")
+ assert _cfn_s3tables_identity(s3tables, bucket_arn) == before
+ finally:
+ _delete_cfn_test_stack(cfn, stack_name)
+
+
+def test_cfn_s3tables_table_create_only_change_under_the_same_name_fails(cfn, s3tables):
+ """The replacement table is refused under the unchanged name and the table stays."""
+ suffix = _uuid_mod.uuid4().hex[:8]
+ stack_name, bucket = f"cfn-s3t-co-{suffix}", f"cfn-s3t-co-{suffix}"
+ try:
+ bucket_arn = _cfn_s3tables_committed_stack(cfn, s3tables, stack_name, bucket)
+ before = _cfn_s3tables_identity(s3tables, bucket_arn)
+
+ cfn.update_stack(StackName=stack_name,
+ TemplateBody=_cfn_s3tables_template(bucket, columns=("id", "v")))
+ stack = _wait_stack(cfn, stack_name)
+ assert stack["StackStatus"] == "UPDATE_ROLLBACK_COMPLETE", stack.get("StackStatusReason")
+ assert "identical name already exists" in _stack_event_reasons(cfn, stack_name)
+ assert _cfn_s3tables_identity(s3tables, bucket_arn) == before
+ finally:
+ _delete_cfn_test_stack(cfn, stack_name)
+
+
+def test_cfn_s3tables_table_rename_removes_the_old_name(cfn, s3tables):
+ """A TableName change leaves the table under the new name only."""
+ suffix = _uuid_mod.uuid4().hex[:8]
+ stack_name, bucket = f"cfn-s3t-mv-{suffix}", f"cfn-s3t-mv-{suffix}"
+ try:
+ bucket_arn = _cfn_s3tables_committed_stack(cfn, s3tables, stack_name, bucket)
+ cfn.update_stack(StackName=stack_name,
+ TemplateBody=_cfn_s3tables_template(bucket, table="t2"))
+ stack = _wait_stack(cfn, stack_name)
+ assert stack["StackStatus"] == "UPDATE_COMPLETE", stack.get("StackStatusReason")
+ tables = s3tables.list_tables(tableBucketARN=bucket_arn)["tables"]
+ assert [t["name"] for t in tables] == ["t2"]
+ finally:
+ _delete_cfn_test_stack(cfn, stack_name)
+
+
# ── AWS::KMS::Key ───────────────────────────────────────────────────────────
# Property names, defaults and update behaviour follow the resource reference:
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-kms-key.html