Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(s3tables): wire CFN table schema through for Iceberg delivery (#1214)
ryan-bennettd6fdfb110fde
diff --git a/ministack/services/cloudformation/provisioners.py b/ministack/services/cloudformation/provisioners.pyindex 6c869efe..7da5455b 100644--- a/ministack/services/cloudformation/provisioners.py+++ b/ministack/services/cloudformation/provisioners.py@@ -5194,7 +5194,20 @@ def _s3tables_table_create(logical_id, props, stack_name):table_name = props.get("TableName", "")bucket_name = bucket_arn.rsplit("/", 1)[-1]location = f"s3://{bucket_name}/{namespace}/{table_name}"- iceberg_metadata = _s3tables._initial_iceberg_metadata(table_name, [], location)+ # IcebergMetadata.IcebergSchema.SchemaFieldList is how CFN (and CDK's+ # Table L1/L2 constructs) declare the table's columns; without parsing it+ # here every CFN-created table ends up with an empty Iceberg schema, which+ # then fails downstream (e.g. a Firehose Iceberg-destination delivery+ # errors with "does not have a column with name ...") even though the+ # template clearly declares one.+ schema_field_list = (+ props.get("IcebergMetadata", {}).get("IcebergSchema", {}).get("SchemaFieldList", [])+ )+ schema_fields = [+ {"name": f["Name"], "type": f.get("Type", "string"), "required": f.get("Required", False)}+ for f in schema_field_list+ ]+ iceberg_metadata = _s3tables._initial_iceberg_metadata(table_name, schema_fields, location)metadata_location = f"s3://{bucket_name}/{namespace}/{table_name}/metadata/v0.metadata.json"table_arn = _s3tables._table_arn(bucket_arn, namespace, table_name)key = _s3tables._table_key(bucket_arn, namespace, table_name)@@ -5205,7 +5218,7 @@ def _s3tables_table_create(logical_id, props, stack_name):"ownerAccountId": get_account_id(),"metadataLocation": metadata_location, "warehouseLocation": location,"_iceberg_metadata": iceberg_metadata, "_metadata_version": 0,- "_schema_fields": [],+ "_schema_fields": schema_fields,}for b in _s3tables._table_buckets.values():if b["arn"] == bucket_arn:diff --git a/tests/test_cfn.py b/tests/test_cfn.pyindex c1d33d67..85fcc848 100644--- a/tests/test_cfn.py+++ b/tests/test_cfn.py@@ -15,6 +15,25 @@ from botocore.exceptions import ClientErrorfrom ministack.services import pipes as _pipes+def _cfn_iceberg_json(path):+ """Hit the S3 Tables Iceberg REST catalog directly (LoadTable etc.) — the+ boto3 s3tables client only exposes the control-plane view, not the actual+ Iceberg schema/metadata a query engine like DuckDB reads."""+ endpoint = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566")+ req = urllib.request.Request(+ f"{endpoint}{path}",+ headers={+ "Authorization": (+ "AWS4-HMAC-SHA256 "+ "Credential=test/20260604/us-east-1/s3tables/aws4_request, "+ "SignedHeaders=host, Signature=test"+ ),+ },+ )+ with urllib.request.urlopen(req) as resp:+ return json.loads(resp.read().decode("utf-8") or "{}")++def _wait_stack(cfn, name, timeout=30):"""Poll until stack reaches terminal status."""deadline = time.time() + timeout@@ -6554,3 +6573,66 @@ def test_cfn_s3tables_resources(cfn, s3tables):cfn.delete_stack(StackName=stack_name)_wait_stack(cfn, stack_name)+++def test_cfn_s3tables_table_schema_from_iceberg_metadata(cfn, s3tables):+ """AWS::S3Tables::Table's IcebergMetadata.IcebergSchema.SchemaFieldList must+ populate the table's actual Iceberg schema — not just be accepted and+ discarded. A table created with an empty schema silently breaks any+ consumer relying on the declared columns (e.g. a Firehose Iceberg+ destination fails to insert with a "does not have a column" error even+ though the CFN template clearly declares one)."""+ template = {+ "AWSTemplateFormatVersion": "2010-09-09",+ "Resources": {+ "Bucket": {+ "Type": "AWS::S3Tables::TableBucket",+ "Properties": {"TableBucketName": "cfn-s3tables-schema-test"},+ },+ "Ns": {+ "Type": "AWS::S3Tables::Namespace",+ "Properties": {+ "TableBucketARN": {"Fn::GetAtt": ["Bucket", "TableBucketARN"]},+ "Namespace": "myns",+ },+ "DependsOn": "Bucket",+ },+ "Table": {+ "Type": "AWS::S3Tables::Table",+ "Properties": {+ "TableBucketARN": {"Fn::GetAtt": ["Bucket", "TableBucketARN"]},+ "Namespace": "myns",+ "TableName": "mytable",+ "OpenTableFormat": "ICEBERG",+ "IcebergMetadata": {+ "IcebergSchema": {+ "SchemaFieldList": [+ {"Id": 1, "Name": "id", "Type": "string", "Required": True},+ {"Id": 2, "Name": "value", "Type": "string"},+ ],+ },+ },+ },+ "DependsOn": "Ns",+ },+ },+ }++ stack_name = "cfn-s3tables-schema-t01"+ try:+ cfn.delete_stack(StackName=stack_name)+ _wait_stack(cfn, stack_name)+ except Exception:+ pass++ cfn.create_stack(StackName=stack_name, TemplateBody=json.dumps(template))+ stack = _wait_stack(cfn, stack_name)+ assert stack["StackStatus"] == "CREATE_COMPLETE", stack.get("StackStatusReason")++ resp = _cfn_iceberg_json(f"/iceberg/v1/namespaces/myns/tables/mytable")+ fields = resp.get("metadata", {}).get("schemas", [{}])[0].get("fields", [])+ field_names = {f["name"] for f in fields}+ assert field_names == {"id", "value"}, f"expected columns id/value, got {field_names}"++ cfn.delete_stack(StackName=stack_name)+ _wait_stack(cfn, stack_name)