Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(s3tables): wire CFN table schema through for Iceberg delivery (#1214)

ryan-bennettd6fdfb110fde
diff --git a/ministack/services/cloudformation/provisioners.py b/ministack/services/cloudformation/provisioners.py
index 6c869efe..7da5455b 100644
--- a/ministack/services/cloudformation/provisioners.py
+++ b/ministack/services/cloudformation/provisioners.py
@@ -5194,7 +5194,20 @@ def _s3tables_table_create(logical_id, props, stack_name):
table_name = props.get("TableName", "")
bucket_name = bucket_arn.rsplit("/", 1)[-1]
location = f"s3://{bucket_name}/{namespace}/{table_name}"
- iceberg_metadata = _s3tables._initial_iceberg_metadata(table_name, [], location)
+ # IcebergMetadata.IcebergSchema.SchemaFieldList is how CFN (and CDK's
+ # Table L1/L2 constructs) declare the table's columns; without parsing it
+ # here every CFN-created table ends up with an empty Iceberg schema, which
+ # then fails downstream (e.g. a Firehose Iceberg-destination delivery
+ # errors with "does not have a column with name ...") even though the
+ # template clearly declares one.
+ schema_field_list = (
+ props.get("IcebergMetadata", {}).get("IcebergSchema", {}).get("SchemaFieldList", [])
+ )
+ schema_fields = [
+ {"name": f["Name"], "type": f.get("Type", "string"), "required": f.get("Required", False)}
+ for f in schema_field_list
+ ]
+ iceberg_metadata = _s3tables._initial_iceberg_metadata(table_name, schema_fields, location)
metadata_location = f"s3://{bucket_name}/{namespace}/{table_name}/metadata/v0.metadata.json"
table_arn = _s3tables._table_arn(bucket_arn, namespace, table_name)
key = _s3tables._table_key(bucket_arn, namespace, table_name)
@@ -5205,7 +5218,7 @@ def _s3tables_table_create(logical_id, props, stack_name):
"ownerAccountId": get_account_id(),
"metadataLocation": metadata_location, "warehouseLocation": location,
"_iceberg_metadata": iceberg_metadata, "_metadata_version": 0,
- "_schema_fields": [],
+ "_schema_fields": schema_fields,
}
for b in _s3tables._table_buckets.values():
if b["arn"] == bucket_arn:
diff --git a/tests/test_cfn.py b/tests/test_cfn.py
index c1d33d67..85fcc848 100644
--- a/tests/test_cfn.py
+++ b/tests/test_cfn.py
@@ -15,6 +15,25 @@ from botocore.exceptions import ClientError
from ministack.services import pipes as _pipes
+def _cfn_iceberg_json(path):
+ """Hit the S3 Tables Iceberg REST catalog directly (LoadTable etc.) — the
+ boto3 s3tables client only exposes the control-plane view, not the actual
+ Iceberg schema/metadata a query engine like DuckDB reads."""
+ endpoint = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566")
+ req = urllib.request.Request(
+ f"{endpoint}{path}",
+ headers={
+ "Authorization": (
+ "AWS4-HMAC-SHA256 "
+ "Credential=test/20260604/us-east-1/s3tables/aws4_request, "
+ "SignedHeaders=host, Signature=test"
+ ),
+ },
+ )
+ with urllib.request.urlopen(req) as resp:
+ return json.loads(resp.read().decode("utf-8") or "{}")
+
+
def _wait_stack(cfn, name, timeout=30):
"""Poll until stack reaches terminal status."""
deadline = time.time() + timeout
@@ -6554,3 +6573,66 @@ def test_cfn_s3tables_resources(cfn, s3tables):
cfn.delete_stack(StackName=stack_name)
_wait_stack(cfn, stack_name)
+
+
+def test_cfn_s3tables_table_schema_from_iceberg_metadata(cfn, s3tables):
+ """AWS::S3Tables::Table's IcebergMetadata.IcebergSchema.SchemaFieldList must
+ populate the table's actual Iceberg schema — not just be accepted and
+ discarded. A table created with an empty schema silently breaks any
+ consumer relying on the declared columns (e.g. a Firehose Iceberg
+ destination fails to insert with a "does not have a column" error even
+ though the CFN template clearly declares one)."""
+ template = {
+ "AWSTemplateFormatVersion": "2010-09-09",
+ "Resources": {
+ "Bucket": {
+ "Type": "AWS::S3Tables::TableBucket",
+ "Properties": {"TableBucketName": "cfn-s3tables-schema-test"},
+ },
+ "Ns": {
+ "Type": "AWS::S3Tables::Namespace",
+ "Properties": {
+ "TableBucketARN": {"Fn::GetAtt": ["Bucket", "TableBucketARN"]},
+ "Namespace": "myns",
+ },
+ "DependsOn": "Bucket",
+ },
+ "Table": {
+ "Type": "AWS::S3Tables::Table",
+ "Properties": {
+ "TableBucketARN": {"Fn::GetAtt": ["Bucket", "TableBucketARN"]},
+ "Namespace": "myns",
+ "TableName": "mytable",
+ "OpenTableFormat": "ICEBERG",
+ "IcebergMetadata": {
+ "IcebergSchema": {
+ "SchemaFieldList": [
+ {"Id": 1, "Name": "id", "Type": "string", "Required": True},
+ {"Id": 2, "Name": "value", "Type": "string"},
+ ],
+ },
+ },
+ },
+ "DependsOn": "Ns",
+ },
+ },
+ }
+
+ stack_name = "cfn-s3tables-schema-t01"
+ try:
+ cfn.delete_stack(StackName=stack_name)
+ _wait_stack(cfn, stack_name)
+ except Exception:
+ pass
+
+ cfn.create_stack(StackName=stack_name, TemplateBody=json.dumps(template))
+ stack = _wait_stack(cfn, stack_name)
+ assert stack["StackStatus"] == "CREATE_COMPLETE", stack.get("StackStatusReason")
+
+ resp = _cfn_iceberg_json(f"/iceberg/v1/namespaces/myns/tables/mytable")
+ fields = resp.get("metadata", {}).get("schemas", [{}])[0].get("fields", [])
+ field_names = {f["name"] for f in fields}
+ assert field_names == {"id", "value"}, f"expected columns id/value, got {field_names}"
+
+ cfn.delete_stack(StackName=stack_name)
+ _wait_stack(cfn, stack_name)