Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
feat(cloudformation): update S3 Tables table buckets and tables in place (#1886)
iot-rocket6e77355e22f4
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e4c19406..5d518141 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -14,6 +14,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).### Fixed- **Cognito — `USER_SRP_AUTH` checks the password** — `PASSWORD_VERIFIER` answered random `SALT` and `SRP_B` values and accepted any response, so a wrong password, a missing `TIMESTAMP` or a disabled or unconfirmed user all got tokens. The challenge now carries SRP-6a parameters with a stable salt per user, and the response is refused with `NotAuthorizedException` unless its signature proves the stored password. `TIMESTAMP` must read `EEE MMM d HH:mm:ss z yyyy` with `UTC` or `GMT` as the zone, the signature is computed over its canonical form, `PASSWORD_CLAIM_SIGNATURE` must be base64, and a temporary password leads to `NEW_PASSWORD_REQUIRED`. A client without `ALLOW_USER_SRP_AUTH`, an unknown user under `LEGACY`, a disabled and an unconfirmed user are refused at `InitiateAuth`. The built-in `PASSWORD_VERIFIER` step of `CUSTOM_WITH_SRP` gets the same check. Contributed by @iot-rocket.- **Cognito — refresh tokens are checked against the pool and client** — `REFRESH_TOKEN_AUTH` (`InitiateAuth`, `AdminInitiateAuth`) and `GetTokensFromRefreshToken` issued tokens for the first user in the pool when the refresh token was malformed, from another pool or of a deleted user, and accepted a token issued to another client; they now answer `NotAuthorizedException` with the message AWS gives for each case, and the refresh token from a SAML or OIDC federated sign-in now refreshes the federated user's session. Contributed by @iot-rocket.+- **CloudFormation — `AWS::S3Tables::TableBucket` and `AWS::S3Tables::Table` update in place** — a stack update re-ran the create, which reset the bucket's `createdAt` and rebuilt the table with a new `createdAt` and its initial metadata location, dropping committed metadata; a maintenance setting or tag change now keeps both, and a create-only table change under an unchanged name fails with the conflict AWS reports. Contributed by @iot-rocket.- **RDS — `DescribeGlobalClusters` lists clusters as `GlobalClusterMember`** — each cluster was a `<GlobalCluster>` element, so SDKs that match the list member name, such as aws-sdk-go-v2, found none. Contributed by @IamYipi.- **Amazon MQ — `DescribeBrokerInstanceOptions` returns `supportedEngineVersions` as strings** — they were `{"name": ...}` objects, which aws-sdk-go-v2 could not deserialize. Contributed by @IamYipi.diff --git a/ministack/services/cloudformation/provisioners.py b/ministack/services/cloudformation/provisioners.pyindex 46055e3a..2f0bbde0 100644--- a/ministack/services/cloudformation/provisioners.py+++ b/ministack/services/cloudformation/provisioners.py@@ -10292,8 +10292,12 @@ def _backup_plan_update(physical_id, old_props, new_props, stack_name,}+def _s3tables_bucket_name(props, stack_name, logical_id):+ return props.get("TableBucketName") or _physical_name(stack_name, logical_id, lowercase=True, max_len=63)++def _s3tables_bucket_create(logical_id, props, stack_name):- name = props.get("TableBucketName") or _physical_name(stack_name, logical_id, lowercase=True, max_len=63)+ name = _s3tables_bucket_name(props, stack_name, logical_id)arn = _s3tables._bucket_arn(name)_s3tables._table_buckets[name] = {"arn": arn, "name": name,@@ -10304,6 +10308,20 @@ def _s3tables_bucket_create(logical_id, props, stack_name):return arn, {"TableBucketARN": arn}+def _s3tables_bucket_update(physical_id, old_props, new_props, stack_name, logical_id=None):+ """TableBucketName replaces the bucket; the other properties keep it."""+ name = physical_id.rsplit("/", 1)[-1]+ replaced = _rename_replacement(+ physical_id, old_props, new_props, stack_name, logical_id,+ _s3tables_bucket_name(new_props, stack_name, logical_id or physical_id),+ name if name in _s3tables._table_buckets else None,+ _s3tables_bucket_create, _s3tables_bucket_delete,+ )+ if replaced is not None:+ return replaced+ return physical_id, {"TableBucketARN": physical_id}++def _s3tables_bucket_delete(physical_id, props):name = physical_id.rsplit("/", 1)[-1]_s3tables._table_buckets.pop(name, None)@@ -10328,10 +10346,30 @@ def _s3tables_namespace_delete(physical_id, props):_s3tables._namespaces.pop(_s3tables._ns_key(bucket_arn, namespace), None)+_S3TABLES_TABLE_CREATE_ONLY = (+ "TableBucketARN", "OpenTableFormat", "IcebergMetadata", "WithoutMetadata",+ "StorageClassConfiguration",+)+++def _s3tables_table_key(props):+ return _s3tables._table_key(+ props.get("TableBucketARN", ""), props.get("Namespace", ""), props.get("TableName", ""))+++def _s3tables_table_attrs(table):+ return {"TableARN": table["tableARN"], "TableBucketARN": table["tableBucketARN"],+ "WarehouseLocation": table["warehouseLocation"], "Namespace": table["namespace"][0],+ "TableName": table["name"]}++def _s3tables_table_create(logical_id, props, stack_name):bucket_arn = props.get("TableBucketARN", "")namespace = props.get("Namespace", "")table_name = props.get("TableName", "")+ key = _s3tables_table_key(props)+ if key in _s3tables._tables:+ raise ValueError("A table with an identical name already exists in the namespace.")bucket_name = bucket_arn.rsplit("/", 1)[-1]location = f"s3://{bucket_name}/{namespace}/{table_name}"# IcebergMetadata.IcebergSchema.SchemaFieldList is how CFN (and CDK's@@ -10350,7 +10388,6 @@ def _s3tables_table_create(logical_id, props, stack_name):iceberg_metadata = _s3tables._initial_iceberg_metadata(table_name, schema_fields, location)metadata_location = f"s3://{bucket_name}/{namespace}/{table_name}/metadata/v0.metadata.json"table_arn = _s3tables._table_arn(bucket_arn, namespace, table_name)- key = _s3tables._table_key(bucket_arn, namespace, table_name)_s3tables._tables[key] = {"name": table_name, "tableARN": table_arn, "namespace": [namespace],"tableBucketARN": bucket_arn, "format": "ICEBERG",@@ -10364,16 +10401,28 @@ def _s3tables_table_create(logical_id, props, stack_name):if b["arn"] == bucket_arn:b["tableCount"] = b.get("tableCount", 0) + 1break- return table_arn, {"TableARN": table_arn, "TableBucketARN": bucket_arn,- "WarehouseLocation": location, "Namespace": namespace,- "TableName": table_name}+ return table_arn, _s3tables_table_attrs(_s3tables._tables[key])+++def _s3tables_table_update(physical_id, old_props, new_props, stack_name):+ """Compaction, SnapshotManagement and Tags keep the table; any other change replaces it."""+ old_key = _s3tables_table_key(old_props)+ table = _s3tables._tables.get(old_key)+ replaced = _rename_replacement(+ physical_id, old_props, new_props, stack_name, None,+ _s3tables_table_key(new_props), old_key if table else None,+ _s3tables_table_create, _s3tables_table_delete,+ )+ if replaced is not None:+ return replaced+ if any(old_props.get(p) != new_props.get(p) for p in _S3TABLES_TABLE_CREATE_ONLY):+ # The replacement is created first, so its unchanged name is refused.+ return _s3tables_table_create(physical_id, new_props, stack_name)+ return physical_id, _s3tables_table_attrs(table)def _s3tables_table_delete(physical_id, props):- bucket_arn = props.get("TableBucketARN", "")- namespace = props.get("Namespace", "")- table_name = props.get("TableName", "")- _s3tables._tables.pop(_s3tables._table_key(bucket_arn, namespace, table_name), None)+ _s3tables._tables.pop(_s3tables_table_key(props), None)# --- Kinesis Data Firehose DeliveryStream ---@@ -11271,9 +11320,18 @@ _RESOURCE_HANDLERS = {"update": _s3_bucket_policy_update,"delete": _s3_bucket_policy_delete,},- "AWS::S3Tables::TableBucket": {"create": _s3tables_bucket_create, "delete": _s3tables_bucket_delete},+ "AWS::S3Tables::TableBucket": {+ "create": _s3tables_bucket_create,+ "update": _s3tables_bucket_update,+ "update_with_logical_id": True,+ "delete": _s3tables_bucket_delete,+ },"AWS::S3Tables::Namespace": {"create": _s3tables_namespace_create, "delete": _s3tables_namespace_delete},- "AWS::S3Tables::Table": {"create": _s3tables_table_create, "delete": _s3tables_table_delete},+ "AWS::S3Tables::Table": {+ "create": _s3tables_table_create,+ "update": _s3tables_table_update,+ "delete": _s3tables_table_delete,+ },"AWS::SQS::Queue": {"create": _sqs_create,"update": _sqs_update,diff --git a/tests/test_cfn.py b/tests/test_cfn.pyindex 2c098599..f5ee5c9c 100644--- a/tests/test_cfn.py+++ b/tests/test_cfn.py@@ -15733,6 +15733,118 @@ def test_cfn_s3tables_table_schema_from_iceberg_metadata(cfn, s3tables):_wait_stack(cfn, stack_name)+def _cfn_s3tables_template(bucket, days=4, target_mb=512, min_snapshots=1, table="t1",+ columns=("id",)):+ return json.dumps({"Resources": {+ "Bucket": {"Type": "AWS::S3Tables::TableBucket", "Properties": {+ "TableBucketName": bucket,+ "UnreferencedFileRemoval": {"Status": "Enabled", "UnreferencedDays": days}}},+ "Ns": {"Type": "AWS::S3Tables::Namespace", "Properties": {+ "TableBucketARN": {"Ref": "Bucket"}, "Namespace": "ns1"}},+ "Table": {"Type": "AWS::S3Tables::Table", "DependsOn": "Ns", "Properties": {+ "TableBucketARN": {"Ref": "Bucket"}, "Namespace": "ns1", "TableName": table,+ "OpenTableFormat": "ICEBERG",+ "IcebergMetadata": {"IcebergSchema": {"SchemaFieldList": [+ {"Name": c, "Type": "int"} for c in columns]}},+ "Compaction": {"Status": "enabled", "TargetFileSizeMB": target_mb},+ "SnapshotManagement": {"Status": "enabled", "MinSnapshotsToKeep": min_snapshots}}},+ }, "Outputs": {"BucketArn": {"Value": {"Ref": "Bucket"}},+ "TableArn": {"Value": {"Ref": "Table"}}}})+++def _cfn_s3tables_committed_stack(cfn, s3tables, stack_name, bucket):+ """A created stack whose table has a committed metadata location; returns the bucket ARN."""+ cfn.create_stack(StackName=stack_name, TemplateBody=_cfn_s3tables_template(bucket))+ stack = _wait_stack(cfn, stack_name)+ assert stack["StackStatus"] == "CREATE_COMPLETE", stack.get("StackStatusReason")+ bucket_arn = _cfn_output(cfn, stack_name, "BucketArn")+ current = s3tables.get_table_metadata_location(+ tableBucketARN=bucket_arn, namespace="ns1", name="t1")+ s3tables.update_table_metadata_location(+ tableBucketARN=bucket_arn, namespace="ns1", name="t1",+ versionToken=current["versionToken"],+ metadataLocation=current["metadataLocation"].replace("v0", "v1"))+ return bucket_arn+++def _cfn_s3tables_identity(s3tables, bucket_arn, table="t1"):+ t = s3tables.get_table(tableBucketARN=bucket_arn, namespace="ns1", name=table)+ return (s3tables.get_table_bucket(tableBucketARN=bucket_arn)["createdAt"],+ t["tableARN"], t["createdAt"], t["metadataLocation"])++[email protected]("change", [+ {"days": 5}, {"target_mb": 256}, {"min_snapshots": 2}])+def test_cfn_s3tables_update_keeps_the_bucket_and_table(cfn, s3tables, change):+ """A mutable property change keeps the bucket and the table with its committed metadata."""+ suffix = _uuid_mod.uuid4().hex[:8]+ stack_name, bucket = f"cfn-s3t-upd-{suffix}", f"cfn-s3t-upd-{suffix}"+ try:+ bucket_arn = _cfn_s3tables_committed_stack(cfn, s3tables, stack_name, bucket)+ before = _cfn_s3tables_identity(s3tables, bucket_arn)+ assert before[3].endswith("/v1.metadata.json")++ cfn.update_stack(StackName=stack_name,+ TemplateBody=_cfn_s3tables_template(bucket, **change))+ stack = _wait_stack(cfn, stack_name)+ assert stack["StackStatus"] == "UPDATE_COMPLETE", stack.get("StackStatusReason")+ assert _cfn_s3tables_identity(s3tables, bucket_arn) == before+ assert _cfn_output(cfn, stack_name, "TableArn") == before[1]+ finally:+ _delete_cfn_test_stack(cfn, stack_name)+++def test_cfn_s3tables_update_is_rolled_back(cfn, s3tables):+ """A rolled-back bucket and table change keeps both as they were."""+ suffix = _uuid_mod.uuid4().hex[:8]+ stack_name, bucket = f"cfn-s3t-rb-{suffix}", f"cfn-s3t-rb-{suffix}"+ try:+ bucket_arn = _cfn_s3tables_committed_stack(cfn, s3tables, stack_name, bucket)+ before = _cfn_s3tables_identity(s3tables, bucket_arn)++ cfn.update_stack(StackName=stack_name, TemplateBody=_cfn_with_failing_resource(+ _cfn_s3tables_template(bucket, days=6, target_mb=128), "Table"))+ stack = _wait_stack(cfn, stack_name)+ assert stack["StackStatus"] == "UPDATE_ROLLBACK_COMPLETE", stack.get("StackStatusReason")+ assert _cfn_s3tables_identity(s3tables, bucket_arn) == before+ finally:+ _delete_cfn_test_stack(cfn, stack_name)+++def test_cfn_s3tables_table_create_only_change_under_the_same_name_fails(cfn, s3tables):+ """The replacement table is refused under the unchanged name and the table stays."""+ suffix = _uuid_mod.uuid4().hex[:8]+ stack_name, bucket = f"cfn-s3t-co-{suffix}", f"cfn-s3t-co-{suffix}"+ try:+ bucket_arn = _cfn_s3tables_committed_stack(cfn, s3tables, stack_name, bucket)+ before = _cfn_s3tables_identity(s3tables, bucket_arn)++ cfn.update_stack(StackName=stack_name,+ TemplateBody=_cfn_s3tables_template(bucket, columns=("id", "v")))+ stack = _wait_stack(cfn, stack_name)+ assert stack["StackStatus"] == "UPDATE_ROLLBACK_COMPLETE", stack.get("StackStatusReason")+ assert "identical name already exists" in _stack_event_reasons(cfn, stack_name)+ assert _cfn_s3tables_identity(s3tables, bucket_arn) == before+ finally:+ _delete_cfn_test_stack(cfn, stack_name)+++def test_cfn_s3tables_table_rename_removes_the_old_name(cfn, s3tables):+ """A TableName change leaves the table under the new name only."""+ suffix = _uuid_mod.uuid4().hex[:8]+ stack_name, bucket = f"cfn-s3t-mv-{suffix}", f"cfn-s3t-mv-{suffix}"+ try:+ bucket_arn = _cfn_s3tables_committed_stack(cfn, s3tables, stack_name, bucket)+ cfn.update_stack(StackName=stack_name,+ TemplateBody=_cfn_s3tables_template(bucket, table="t2"))+ stack = _wait_stack(cfn, stack_name)+ assert stack["StackStatus"] == "UPDATE_COMPLETE", stack.get("StackStatusReason")+ tables = s3tables.list_tables(tableBucketARN=bucket_arn)["tables"]+ assert [t["name"] for t in tables] == ["t2"]+ finally:+ _delete_cfn_test_stack(cfn, stack_name)++# ── AWS::KMS::Key ───────────────────────────────────────────────────────────# Property names, defaults and update behaviour follow the resource reference:# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-kms-key.html