Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
"""S3 Tables service tests — round-trip coverage for the 14 control-plane
operations the service ships in 1.3.50, plus a multi-tenancy isolation check.
Operations covered:
CreateTableBucket, ListTableBuckets, GetTableBucket, DeleteTableBucket
CreateNamespace, ListNamespaces, GetNamespace, DeleteNamespace
CreateTable, ListTables, GetTable, DeleteTable
GetTableMetadataLocation, UpdateTableMetadataLocation
Shapes verified against `botocore.data.s3tables.2024-12-01.service-2`.
"""
import contextlib
import json
import os
import urllib.error
import urllib.parse
import urllib.request
import uuid as _uuid_mod
import boto3
import pytest
from botocore.config import Config
from botocore.exceptions import ClientError
_ENDPOINT = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566")
def _iceberg_json(path, method="GET", payload=None, region_name=None, authorization=None):
data = json.dumps(payload).encode("utf-8") if payload is not None else None
headers = {"Content-Type": "application/json"}
if authorization:
headers["Authorization"] = authorization
elif region_name:
headers["Authorization"] = (
"AWS4-HMAC-SHA256 "
f"Credential=test/20260604/{region_name}/s3tables/aws4_request, "
"SignedHeaders=host, Signature=test"
)
req = urllib.request.Request(
f"{_ENDPOINT}{path}",
data=data,
method=method,
headers=headers,
)
with urllib.request.urlopen(req) as resp:
return json.loads(resp.read().decode("utf-8") or "{}")
def _make_s3tables_client(access_key="test", region_name="us-east-1"):
return boto3.client(
"s3tables",
endpoint_url=_ENDPOINT,
aws_access_key_id=access_key,
aws_secret_access_key="test",
region_name=region_name,
config=Config(retries={"mode": "standard"}),
)
@pytest.fixture(scope="session")
def s3tables():
return _make_s3tables_client()
# ── Table bucket lifecycle ──────────────────────────────────
def test_s3tables_create_list_get_delete_bucket(s3tables):
name = f"tb-bucket-{_uuid_mod.uuid4().hex[:8]}"
created = s3tables.create_table_bucket(name=name)
assert "arn" in created
arn = created["arn"]
assert name in arn
assert arn.startswith("arn:aws:s3tables:")
listed = s3tables.list_table_buckets()
names = {b.get("name") for b in listed.get("tableBuckets", [])}
assert name in names, f"created bucket {name!r} not in ListTableBuckets"
got = s3tables.get_table_bucket(tableBucketARN=arn)
assert got.get("name") == name
assert got.get("arn") == arn
s3tables.delete_table_bucket(tableBucketARN=arn)
with pytest.raises(ClientError) as exc:
s3tables.get_table_bucket(tableBucketARN=arn)
assert exc.value.response["Error"]["Code"] in ("NotFoundException", "404")
def test_s3tables_table_bucket_provisions_backing_s3_bucket(s3tables):
"""Creating a table bucket must automatically provision a same-named S3 bucket
so that data-plane writes (Parquet files, manifests) have storage to land in —
mirroring real AWS where S3 Tables manages underlying storage transparently."""
import boto3
from botocore.config import Config
name = f"tb-backing-{_uuid_mod.uuid4().hex[:8]}"
arn = s3tables.create_table_bucket(name=name)["arn"]
try:
s3_client = boto3.client(
"s3",
endpoint_url=_ENDPOINT,
aws_access_key_id="test",
aws_secret_access_key="test",
region_name="us-east-1",
config=Config(retries={"mode": "standard"}),
)
# The backing bucket must be accessible via the S3 API
s3_client.head_bucket(Bucket=name)
# And must accept object writes (simulating a Parquet data file)
s3_client.put_object(Bucket=name, Key="ns/tbl/data/part-0.parquet", Body=b"parquet")
obj = s3_client.get_object(Bucket=name, Key="ns/tbl/data/part-0.parquet")
assert obj["Body"].read() == b"parquet"
# Deleting the table bucket must also remove the backing S3 bucket
s3tables.delete_table_bucket(tableBucketARN=arn)
from botocore.exceptions import ClientError as _CE
with pytest.raises(_CE) as exc:
s3_client.head_bucket(Bucket=name)
assert exc.value.response["Error"]["Code"] in ("404", "NoSuchBucket")
except Exception:
try:
s3tables.delete_table_bucket(tableBucketARN=arn)
except Exception:
pass
raise
def test_s3tables_get_bucket_missing_returns_not_found(s3tables):
fake_arn = "arn:aws:s3tables:us-east-1:000000000000:bucket/does-not-exist-xyz"
with pytest.raises(ClientError) as exc:
s3tables.get_table_bucket(tableBucketARN=fake_arn)
assert exc.value.response["Error"]["Code"] in ("NotFoundException", "404")
def test_s3tables_bucket_arn_scope_does_not_fallback_to_local_bucket(s3tables):
bucket_name = f"tb-scope-{_uuid_mod.uuid4().hex[:8]}"
arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
s3tables.create_namespace(tableBucketARN=arn, namespace=[ns])
s3tables.create_table(
tableBucketARN=arn,
namespace=ns,
name=table,
format="ICEBERG",
metadata={"iceberg": {"schema": {"fields": [{"name": "id", "type": "long"}]}}},
)
wrong_region = arn.replace(":us-east-1:", ":us-west-2:")
wrong_account = arn.replace(":000000000000:", ":111111111111:")
wrong_service = arn.replace(":s3tables:", ":s3:")
wrong_resource = arn.replace(":bucket/", ":table/")
for bad_ref in (wrong_region, wrong_account, wrong_service, wrong_resource):
with pytest.raises(ClientError) as exc:
s3tables.get_table_bucket(tableBucketARN=bad_ref)
assert exc.value.response["Error"]["Code"] in ("NotFoundException", "404")
with pytest.raises(ClientError) as exc:
s3tables.create_namespace(tableBucketARN=bad_ref, namespace=[f"ns_{_uuid_mod.uuid4().hex[:6]}"])
assert exc.value.response["Error"]["Code"] in ("NotFoundException", "404")
with pytest.raises(ClientError) as exc:
s3tables.list_tables(tableBucketARN=bad_ref)
assert exc.value.response["Error"]["Code"] in ("NotFoundException", "404")
with pytest.raises(ClientError) as exc:
s3tables.get_table(tableBucketARN=bad_ref, namespace=ns, name=table)
assert exc.value.response["Error"]["Code"] in ("NotFoundException", "404")
finally:
try:
s3tables.delete_table(tableBucketARN=arn, namespace=ns, name=table)
except Exception:
pass
try:
s3tables.delete_namespace(tableBucketARN=arn, namespace=ns)
except Exception:
pass
s3tables.delete_table_bucket(tableBucketARN=arn)
# ── Namespace lifecycle ─────────────────────────────────────
def test_s3tables_create_list_get_delete_namespace(s3tables):
bucket_name = f"tb-ns-{_uuid_mod.uuid4().hex[:8]}"
arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
try:
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
created = s3tables.create_namespace(tableBucketARN=arn, namespace=[ns])
assert created.get("namespace") == [ns] or created.get("namespace") == ns
listed = s3tables.list_namespaces(tableBucketARN=arn)
ns_values = []
for entry in listed.get("namespaces", []):
n = entry.get("namespace")
ns_values.append(n[0] if isinstance(n, list) else n)
assert ns in ns_values
got = s3tables.get_namespace(tableBucketARN=arn, namespace=ns)
got_ns = got.get("namespace")
assert (got_ns[0] if isinstance(got_ns, list) else got_ns) == ns
s3tables.delete_namespace(tableBucketARN=arn, namespace=ns)
with pytest.raises(ClientError) as exc:
s3tables.get_namespace(tableBucketARN=arn, namespace=ns)
assert exc.value.response["Error"]["Code"] in ("NotFoundException", "404")
finally:
s3tables.delete_table_bucket(tableBucketARN=arn)
# ── Table lifecycle ─────────────────────────────────────────
def test_s3tables_create_list_get_delete_table(s3tables):
bucket_name = f"tb-tbl-{_uuid_mod.uuid4().hex[:8]}"
arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
try:
s3tables.create_namespace(tableBucketARN=arn, namespace=[ns])
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
created = s3tables.create_table(
tableBucketARN=arn, namespace=ns, name=table, format="ICEBERG",
metadata={
"iceberg": {
"schema": {
"fields": [
{"name": "id", "type": "long", "required": True},
{"name": "value", "type": "string"},
]
}
}
},
)
assert "tableARN" in created
table_arn = created["tableARN"]
assert ns in table_arn and table in table_arn
listed = s3tables.list_tables(tableBucketARN=arn)
table_names = {t.get("name") for t in listed.get("tables", [])}
assert table in table_names
got = s3tables.get_table(tableBucketARN=arn, namespace=ns, name=table)
assert got.get("name") == table
assert got.get("format") == "ICEBERG"
s3tables.delete_table(tableBucketARN=arn, namespace=ns, name=table)
with pytest.raises(ClientError) as exc:
s3tables.get_table(tableBucketARN=arn, namespace=ns, name=table)
assert exc.value.response["Error"]["Code"] in ("NotFoundException", "404")
finally:
try:
s3tables.delete_namespace(tableBucketARN=arn, namespace=ns)
except Exception:
pass
s3tables.delete_table_bucket(tableBucketARN=arn)
# ── Metadata location round-trip ────────────────────────────
def test_s3tables_get_update_table_metadata_location(s3tables):
bucket_name = f"tb-md-{_uuid_mod.uuid4().hex[:8]}"
arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
s3tables.create_namespace(tableBucketARN=arn, namespace=[ns])
s3tables.create_table(
tableBucketARN=arn, namespace=ns, name=table, format="ICEBERG",
metadata={"iceberg": {"schema": {"fields": [{"name": "id", "type": "long"}]}}},
)
got = s3tables.get_table_metadata_location(
tableBucketARN=arn, namespace=ns, name=table)
assert "metadataLocation" in got
token = got.get("versionToken", "")
new_loc = f"s3://{bucket_name}/{ns}/{table}/metadata/v1.metadata.json"
updated = s3tables.update_table_metadata_location(
tableBucketARN=arn, namespace=ns, name=table,
versionToken=token, metadataLocation=new_loc,
)
assert updated.get("metadataLocation") == new_loc
got2 = s3tables.get_table_metadata_location(
tableBucketARN=arn, namespace=ns, name=table)
assert got2.get("metadataLocation") == new_loc
finally:
try:
s3tables.delete_table(tableBucketARN=arn, namespace=ns, name=table)
except Exception:
pass
try:
s3tables.delete_namespace(tableBucketARN=arn, namespace=ns)
except Exception:
pass
s3tables.delete_table_bucket(tableBucketARN=arn)
# ── Multi-tenancy isolation ─────────────────────────────────
def test_s3tables_buckets_are_account_scoped(s3tables):
"""Same bucket name under two different account IDs must not collide.
Multi-tenancy is enforced by the SigV4 access-key-derived account ID; we
swap clients with 12-digit access keys and assert ListTableBuckets returns
only the caller's buckets."""
acct_a = "111111111111"
acct_b = "222222222222"
name = f"shared-{_uuid_mod.uuid4().hex[:6]}"
client_a = _make_s3tables_client(access_key=acct_a)
client_b = _make_s3tables_client(access_key=acct_b)
arn_a = client_a.create_table_bucket(name=name)["arn"]
arn_b = client_b.create_table_bucket(name=name)["arn"]
try:
assert acct_a in arn_a
assert acct_b in arn_b
assert arn_a != arn_b
names_a = {b.get("name") for b in client_a.list_table_buckets().get("tableBuckets", [])}
names_b = {b.get("name") for b in client_b.list_table_buckets().get("tableBuckets", [])}
assert name in names_a
assert name in names_b
# Cross-account access must not see the other tenant's bucket.
with pytest.raises(ClientError):
client_a.get_table_bucket(tableBucketARN=arn_b)
with pytest.raises(ClientError):
client_b.get_table_bucket(tableBucketARN=arn_a)
finally:
try:
client_a.delete_table_bucket(tableBucketARN=arn_a)
except Exception:
pass
try:
client_b.delete_table_bucket(tableBucketARN=arn_b)
except Exception:
pass
def test_s3tables_buckets_are_region_scoped():
name = f"regional-{_uuid_mod.uuid4().hex[:6]}"
east = _make_s3tables_client(region_name="us-east-1")
west = _make_s3tables_client(region_name="us-west-2")
east_arn = east.create_table_bucket(name=name)["arn"]
west_arn = west.create_table_bucket(name=name)["arn"]
try:
assert ":us-east-1:" in east_arn
assert ":us-west-2:" in west_arn
assert east_arn != west_arn
east_arns = {b.get("arn") for b in east.list_table_buckets().get("tableBuckets", [])}
west_arns = {b.get("arn") for b in west.list_table_buckets().get("tableBuckets", [])}
assert east_arn in east_arns
assert west_arn not in east_arns
assert west_arn in west_arns
assert east_arn not in west_arns
with pytest.raises(ClientError) as exc:
east.get_table_bucket(tableBucketARN=west_arn)
assert exc.value.response["Error"]["Code"] in ("NotFoundException", "404")
finally:
try:
east.delete_table_bucket(tableBucketARN=east_arn)
except Exception:
pass
try:
west.delete_table_bucket(tableBucketARN=west_arn)
except Exception:
pass
def test_s3tables_iceberg_catalog_spans_control_plane_regions():
west = _make_s3tables_client(region_name="us-west-2")
bucket_name = f"iceberg-west-{_uuid_mod.uuid4().hex[:6]}"
bucket_arn = west.create_table_bucket(name=bucket_name)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
iceberg_table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
west.create_namespace(tableBucketARN=bucket_arn, namespace=[ns])
west.create_table(
tableBucketARN=bucket_arn,
namespace=ns,
name=table,
format="ICEBERG",
metadata={"iceberg": {"schema": {"fields": [{"name": "id", "type": "long"}]}}},
)
namespaces = _iceberg_json("/iceberg/v1/catalog/namespaces")
assert [ns] in namespaces.get("namespaces", [])
tables = _iceberg_json(f"/iceberg/v1/catalog/namespaces/{ns}/tables")
assert {"namespace": [ns], "name": table} in tables.get("identifiers", [])
loaded = _iceberg_json(f"/iceberg/v1/catalog/namespaces/{ns}/tables/{table}")
assert loaded.get("metadata-location", "").startswith(f"s3://{bucket_name}/{ns}/{table}/")
bearer_loaded = _iceberg_json(
f"/iceberg/v1/catalog/namespaces/{ns}/tables/{table}",
authorization="Bearer test-token",
)
assert bearer_loaded.get("metadata-location", "").startswith(f"s3://{bucket_name}/{ns}/{table}/")
_iceberg_json(
f"/iceberg/v1/catalog/namespaces/{ns}/tables",
method="POST",
payload={
"name": iceberg_table,
"schema": {"type": "struct", "fields": [{"id": 1, "name": "id", "type": "long"}]},
},
)
got = west.get_table(tableBucketARN=bucket_arn, namespace=ns, name=iceberg_table)
assert got["name"] == iceberg_table
finally:
for candidate in (table, iceberg_table):
try:
west.delete_table(tableBucketARN=bucket_arn, namespace=ns, name=candidate)
except Exception:
pass
try:
west.delete_namespace(tableBucketARN=bucket_arn, namespace=ns)
except Exception:
pass
west.delete_table_bucket(tableBucketARN=bucket_arn)
@pytest.mark.parametrize("headers, query, cross_region", [
({"authorization": "AWS AKIAEXAMPLE:sig"}, {}, False),
({}, {"x-amz-credential": ["AKIAEXAMPLE/20260101/us-west-2/s3tables/aws4_request"]}, False),
({"authorization": "AWS4-HMAC-SHA256 Credential=AKIAEXAMPLE/20260101/us-west-2/s3tables/aws4_request, "
"SignedHeaders=host, Signature=abc"}, {}, False),
({}, {}, True),
], ids=["sigv2", "lowercase-query", "sigv4", "unsigned"])
def test_s3tables_iceberg_auth_holds_a_named_key_to_its_region(monkeypatch, headers, query, cross_region):
"""Under AUTH a call that names any key serves its own region, the one its checks resolve."""
import ministack.app as app
from ministack.services import s3tables
monkeypatch.setattr(app, "AUTH", True)
assert s3tables._iceberg_allows_cross_region(headers, query) is cross_region
def test_s3tables_iceberg_catalog_prefers_signed_region_for_duplicate_names():
east = _make_s3tables_client(region_name="us-east-1")
west = _make_s3tables_client(region_name="us-west-2")
east_bucket = f"iceberg-east-{_uuid_mod.uuid4().hex[:6]}"
west_bucket = f"iceberg-west-{_uuid_mod.uuid4().hex[:6]}"
east_arn = east.create_table_bucket(name=east_bucket)["arn"]
west_arn = west.create_table_bucket(name=west_bucket)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
for client, bucket_arn in ((east, east_arn), (west, west_arn)):
client.create_namespace(tableBucketARN=bucket_arn, namespace=[ns])
client.create_table(
tableBucketARN=bucket_arn,
namespace=ns,
name=table,
format="ICEBERG",
metadata={"iceberg": {"schema": {"fields": [{"name": "id", "type": "long"}]}}},
)
east_loaded = _iceberg_json(
f"/iceberg/v1/catalog/namespaces/{ns}/tables/{table}",
region_name="us-east-1",
)
west_loaded = _iceberg_json(
f"/iceberg/v1/catalog/namespaces/{ns}/tables/{table}",
region_name="us-west-2",
)
assert east_loaded.get("metadata-location", "").startswith(f"s3://{east_bucket}/{ns}/{table}/")
assert west_loaded.get("metadata-location", "").startswith(f"s3://{west_bucket}/{ns}/{table}/")
with pytest.raises(urllib.error.HTTPError) as exc:
_iceberg_json(
f"/iceberg/v1/catalog/namespaces/{ns}/tables/{table}",
region_name="us-east-2",
)
assert exc.value.code == 404
finally:
for client, bucket_arn in ((east, east_arn), (west, west_arn)):
try:
client.delete_table(tableBucketARN=bucket_arn, namespace=ns, name=table)
except Exception:
pass
try:
client.delete_namespace(tableBucketARN=bucket_arn, namespace=ns)
except Exception:
pass
try:
client.delete_table_bucket(tableBucketARN=bucket_arn)
except Exception:
pass
def test_iceberg_rest_error_uses_spec_envelope():
"""The /iceberg REST surface must return the Iceberg REST OpenAPI ErrorModel
({"error": {message, type, code}}), not the AWS {"__type"} shape. A LoadTable
on a missing table must be a proper NoSuchTableException so spec-compliant
writers (DuckDB, Spark) proceed to create it instead of aborting."""
with pytest.raises(urllib.error.HTTPError) as exc:
_iceberg_json("/iceberg/v1/catalog/namespaces/nope/tables/missing")
assert exc.value.code == 404
body = json.loads(exc.value.read().decode("utf-8"))
assert "__type" not in body
assert body["error"]["type"] == "NoSuchTableException"
assert body["error"]["code"] == 404
assert body["error"]["message"]
def test_s3tables_iceberg_routes_without_prefix_are_unknown(s3tables):
"""Every route but /v1/config takes the table bucket ARN prefix; real AWS
answers one without it 404 <UnknownOperationException/>."""
bucket_arn = s3tables.create_table_bucket(name=f"tb-noprefix-{_uuid_mod.uuid4().hex[:6]}")["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
s3tables.create_namespace(tableBucketARN=bucket_arn, namespace=[ns])
s3tables.create_table(
tableBucketARN=bucket_arn, namespace=ns, name=table, format="ICEBERG",
metadata={"iceberg": {"schema": {"fields": [{"name": "id", "type": "long"}]}}},
)
for method, path in (
("GET", "/iceberg/v1/namespaces"),
("GET", f"/iceberg/v1/namespaces/{ns}/tables"),
("GET", f"/iceberg/v1/namespaces/{ns}/tables/{table}"),
("POST", f"/iceberg/v1/namespaces/{ns}/tables"),
("POST", "/iceberg/v1/transactions/commit"),
):
req = urllib.request.Request(f"{_ENDPOINT}{path}", method=method, data=b"{}" if method == "POST" else None)
with pytest.raises(urllib.error.HTTPError) as exc:
urllib.request.urlopen(req)
assert (exc.value.code, exc.value.read()) == (404, b"<UnknownOperationException/>")
loaded = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")
assert loaded["metadata-location"].startswith(f"s3://{bucket_arn.rsplit('/', 1)[1]}/{ns}/{table}/")
finally:
s3tables.delete_table(tableBucketARN=bucket_arn, namespace=ns, name=table)
s3tables.delete_namespace(tableBucketARN=bucket_arn, namespace=ns)
s3tables.delete_table_bucket(tableBucketARN=bucket_arn)
def test_s3tables_iceberg_config_returns_s3_defaults():
"""GET /iceberg/v1/config hands back the url-encoded table bucket ARN as the
prefix in defaults, as AWS does, plus the S3 connection defaults."""
arn = "arn:aws:s3tables:us-east-1:000000000000:bucket/cfg-bucket"
resp = _iceberg_json(f"/iceberg/v1/config?warehouse={_arn_prefix(arn)}")
defaults = resp["defaults"]
assert defaults["prefix"] == _arn_prefix(arn)
assert resp["overrides"] == {}
for key in ("s3.endpoint", "s3.access-key-id", "s3.secret-access-key", "s3.path-style-access", "client.region"):
assert key in defaults
@pytest.mark.parametrize("query", ["", "?warehouse=arn:aws:s3tables:us-east-1:000000000000:bucket"])
def test_s3tables_iceberg_config_needs_a_table_bucket_arn(query):
status, body = _iceberg_call(f"/iceberg/v1/config{query}")
assert (status, body) == (400, {"error": {
"message": "The specified table bucket ARN is not valid.", "type": "bad_request", "code": 400}})
def test_s3tables_iceberg_load_table_includes_region(s3tables):
"""LoadTable response config must include s3.region and client.region so
DuckDB / Spark can route S3 data-file requests to the right region."""
bucket_name = f"tb-region-{_uuid_mod.uuid4().hex[:6]}"
bucket_arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
s3tables.create_namespace(tableBucketARN=bucket_arn, namespace=[ns])
s3tables.create_table(tableBucketARN=bucket_arn, namespace=ns, name=table, format="ICEBERG")
resp = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")
config = resp.get("config", {})
assert "s3.region" in config, "s3.region missing from LoadTable config"
assert "client.region" in config, "client.region missing from LoadTable config"
finally:
try:
s3tables.delete_table(tableBucketARN=bucket_arn, namespace=ns, name=table)
except Exception:
pass
try:
s3tables.delete_namespace(tableBucketARN=bucket_arn, namespace=ns)
except Exception:
pass
s3tables.delete_table_bucket(tableBucketARN=bucket_arn)
def test_s3tables_iceberg_transactions_commit(s3tables):
"""POST /iceberg/v1/transactions/commit — DuckDB uses this endpoint for
atomic multi-table commits when writing data."""
bucket_name = f"tb-txn-{_uuid_mod.uuid4().hex[:6]}"
bucket_arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
s3tables.create_namespace(tableBucketARN=bucket_arn, namespace=[ns])
s3tables.create_table(tableBucketARN=bucket_arn, namespace=ns, name=table, format="ICEBERG")
snapshot_id = 1234567890
resp = _iceberg_json(
f"/iceberg/v1/{_arn_prefix(bucket_arn)}/transactions/commit",
method="POST",
payload={
"table-changes": [
{
"identifier": {"namespace": [ns], "name": table},
"requirements": [],
"updates": [
{
"action": "add-snapshot",
"snapshot": {
"snapshot-id": snapshot_id,
"sequence-number": 1,
"timestamp-ms": 1700000000000,
"manifest-list": f"s3://{bucket_name}/{ns}/{table}/metadata/snap.avro",
"summary": {"operation": "append"},
},
},
{
"action": "set-snapshot-ref",
"ref-name": "main",
"type": "branch",
"snapshot-id": snapshot_id,
},
],
}
]
},
)
# A successful commit returns 200 with an empty body (or minimal body)
assert resp is not None
# Verify the snapshot was recorded in the table metadata
loaded = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")
metadata = loaded.get("metadata", {})
snap_ids = [s.get("snapshot-id") for s in metadata.get("snapshots", [])]
assert snapshot_id in snap_ids, f"snapshot {snapshot_id} not found after commit"
assert metadata.get("current-snapshot-id") == snapshot_id
finally:
try:
s3tables.delete_table(tableBucketARN=bucket_arn, namespace=ns, name=table)
except Exception:
pass
try:
s3tables.delete_namespace(tableBucketARN=bucket_arn, namespace=ns)
except Exception:
pass
s3tables.delete_table_bucket(tableBucketARN=bucket_arn)
def test_s3tables_iceberg_upgrade_format_version(s3tables):
"""A commit carrying the Iceberg REST ``upgrade-format-version`` action
(what Spark sends for ``.tableProperty("format-version", "3")`` on an
existing table) moves the table's metadata to that version; re-asserting
the current version is a no-op and a downgrade is refused with 400.
Real S3 Tables supports Iceberg format version 3."""
bucket_name = f"tb-fv-{_uuid_mod.uuid4().hex[:6]}"
bucket_arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
s3tables.create_namespace(tableBucketARN=bucket_arn, namespace=[ns])
s3tables.create_table(tableBucketARN=bucket_arn, namespace=ns, name=table, format="ICEBERG")
loaded = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")
assert loaded["metadata"]["format-version"] == 2
resp = _iceberg_json(
f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}", method="POST",
payload={"requirements": [],
"updates": [{"action": "upgrade-format-version", "format-version": 3}]},
)
assert resp["metadata"]["format-version"] == 3
loaded = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")
assert loaded["metadata"]["format-version"] == 3
# Re-asserting the version the table already has is a no-op commit.
resp = _iceberg_json(
f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}", method="POST",
payload={"requirements": [],
"updates": [{"action": "upgrade-format-version", "format-version": 3}]},
)
assert resp["metadata"]["format-version"] == 3
# A downgrade is refused, and the table keeps its version.
with pytest.raises(urllib.error.HTTPError) as exc_info:
_iceberg_json(
f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}", method="POST",
payload={"requirements": [],
"updates": [{"action": "upgrade-format-version", "format-version": 2}]},
)
assert exc_info.value.code == 400
assert b"downgrade" in exc_info.value.read()
loaded = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")
assert loaded["metadata"]["format-version"] == 3
finally:
try:
s3tables.delete_table(tableBucketARN=bucket_arn, namespace=ns, name=table)
except Exception:
pass
try:
s3tables.delete_namespace(tableBucketARN=bucket_arn, namespace=ns)
except Exception:
pass
s3tables.delete_table_bucket(tableBucketARN=bucket_arn)
def test_s3tables_iceberg_create_table_rejects_resend_instead_of_wiping_data(s3tables):
"""CreateTable is not idempotent per the Iceberg REST spec -- a resent create
(e.g. a client retry after a lost response to a create that already landed)
must be rejected with a conflict, matching the S3 Tables control-plane path's
existing 409 behaviour. Before this guard, a resend silently replaced the
table's metadata with a fresh empty one, discarding any snapshots/schema
already committed -- worse than a crash, since nothing signals data was lost."""
bucket_name = f"tb-createretry-{_uuid_mod.uuid4().hex[:6]}"
bucket_arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
s3tables.create_namespace(tableBucketARN=bucket_arn, namespace=[ns])
_iceberg_json(
f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables",
method="POST",
payload={"name": table, "schema": {"type": "struct", "schema-id": 0, "fields": []}},
)
# Commit a snapshot so there's real state a resent create could destroy.
snapshot_id = 42
_iceberg_json(
f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}",
method="POST",
payload={
"requirements": [],
"updates": [
{
"action": "add-snapshot",
"snapshot": {
"snapshot-id": snapshot_id,
"sequence-number": 1,
"timestamp-ms": 1700000000000,
"manifest-list": f"s3://{bucket_name}/{ns}/{table}/metadata/snap.avro",
"summary": {"operation": "append"},
},
},
],
},
)
# Resend the identical create, simulating a retry of a create whose first
# response was lost.
try:
_iceberg_json(
f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables",
method="POST",
payload={"name": table, "schema": {"type": "struct", "schema-id": 0, "fields": []}},
)
assert False, "expected a 409 conflict on resent CreateTable"
except urllib.error.HTTPError as e:
assert e.code == 409
body = json.loads(e.read().decode("utf-8"))
assert body["error"]["type"] == "AlreadyExistsException"
# The original snapshot must still be there -- not wiped by the resend.
loaded = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")
snap_ids = [s.get("snapshot-id") for s in loaded["metadata"]["snapshots"]]
assert snapshot_id in snap_ids, "resent create-table must not wipe existing table state"
finally:
try:
s3tables.delete_table(tableBucketARN=bucket_arn, namespace=ns, name=table)
except Exception:
pass
try:
s3tables.delete_namespace(tableBucketARN=bucket_arn, namespace=ns)
except Exception:
pass
s3tables.delete_table_bucket(tableBucketARN=bucket_arn)
def test_s3tables_iceberg_add_schema_advances_last_column_id(s3tables):
"""Each add-schema commit must bump last-column-id to the new schema's highest
field ID, so a client allocating the *next* column's ID (e.g. DuckDB's
ALTER TABLE ... ADD COLUMN) doesn't collide with a previous add-schema's fields."""
bucket_name = f"tb-lcid-{_uuid_mod.uuid4().hex[:6]}"
bucket_arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
s3tables.create_namespace(tableBucketARN=bucket_arn, namespace=[ns])
s3tables.create_table(tableBucketARN=bucket_arn, namespace=ns, name=table, format="ICEBERG")
_iceberg_json(
f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}",
method="POST",
payload={
"requirements": [],
"updates": [
{
"action": "add-schema",
"schema": {
"type": "struct", "schema-id": 1,
"fields": [{"id": 1, "name": "colA", "required": False, "type": "string"}],
},
},
{"action": "set-current-schema", "schema-id": 1},
],
},
)
first = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")
assert first["metadata"]["last-column-id"] == 1
_iceberg_json(
f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}",
method="POST",
payload={
"requirements": [],
"updates": [
{
"action": "add-schema",
"schema": {
"type": "struct", "schema-id": 2,
"fields": [
{"id": 1, "name": "colA", "required": False, "type": "string"},
{"id": 2, "name": "colB", "required": False, "type": "long"},
],
},
},
{"action": "set-current-schema", "schema-id": 2},
],
},
)
second = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")
metadata = second["metadata"]
assert metadata["last-column-id"] == 2
current_schema = next(s for s in metadata["schemas"] if s["schema-id"] == metadata["current-schema-id"])
field_ids = [f["id"] for f in current_schema["fields"]]
assert field_ids == [1, 2], "field IDs must not collide across separate add-schema commits"
finally:
try:
s3tables.delete_table(tableBucketARN=bucket_arn, namespace=ns, name=table)
except Exception:
pass
try:
s3tables.delete_namespace(tableBucketARN=bucket_arn, namespace=ns)
except Exception:
pass
s3tables.delete_table_bucket(tableBucketARN=bucket_arn)
def test_s3tables_iceberg_add_spec_uses_real_wire_action_name(s3tables):
"""The Iceberg REST spec's real action name for adding a partition spec is
"add-spec" (what duckdb-iceberg actually sends) — not "add-partition-spec",
a non-standard name only some hand-rolled callers use. Both must work."""
bucket_name = f"tb-spec-{_uuid_mod.uuid4().hex[:6]}"
bucket_arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
s3tables.create_namespace(tableBucketARN=bucket_arn, namespace=[ns])
s3tables.create_table(tableBucketARN=bucket_arn, namespace=ns, name=table, format="ICEBERG")
_iceberg_json(
f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}",
method="POST",
payload={
"requirements": [],
"updates": [
{
"action": "add-spec",
"spec": {
"spec-id": 1,
"fields": [{"source-id": 1, "field-id": 1000, "name": "day_col", "transform": "day"}],
},
},
{"action": "set-default-spec", "spec-id": 1},
],
},
)
loaded = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")
metadata = loaded["metadata"]
spec_ids = [s["spec-id"] for s in metadata["partition-specs"]]
assert 1 in spec_ids, "add-spec must append the new partition spec"
assert metadata["default-spec-id"] == 1
finally:
try:
s3tables.delete_table(tableBucketARN=bucket_arn, namespace=ns, name=table)
except Exception:
pass
try:
s3tables.delete_namespace(tableBucketARN=bucket_arn, namespace=ns)
except Exception:
pass
s3tables.delete_table_bucket(tableBucketARN=bucket_arn)
def test_s3tables_iceberg_create_table_honors_requested_partition_spec(s3tables):
"""The create-table endpoint's response must echo back the partition spec the
client actually asked for, not a hardcoded empty one -- clients (e.g.
duckdb-iceberg) initialize their own local table state from this response, so an
empty echo here gets asserted right back at the server on the very next commit,
permanently losing the partition spec even though it was in the original request."""
bucket_name = f"tb-createspec-{_uuid_mod.uuid4().hex[:6]}"
bucket_arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
s3tables.create_namespace(tableBucketARN=bucket_arn, namespace=[ns])
created = _iceberg_json(
f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables",
method="POST",
payload={
"name": table,
"schema": {
"type": "struct", "schema-id": 0,
"fields": [{"id": 1, "name": "ts", "required": False, "type": "timestamp"}],
},
"partition-spec": {
"spec-id": 0,
"fields": [{"source-id": 1, "field-id": 1000, "name": "day_ts", "transform": "day"}],
},
},
)
metadata = created["metadata"]
spec = next(s for s in metadata["partition-specs"] if s["spec-id"] == metadata["default-spec-id"])
assert spec["fields"], "create-table response must echo back the requested partition spec, not an empty one"
assert spec["fields"][0]["transform"] == "day"
finally:
try:
s3tables.delete_table(tableBucketARN=bucket_arn, namespace=ns, name=table)
except Exception:
pass
try:
s3tables.delete_namespace(tableBucketARN=bucket_arn, namespace=ns)
except Exception:
pass
s3tables.delete_table_bucket(tableBucketARN=bucket_arn)
def test_s3tables_iceberg_add_schema_is_idempotent_by_schema_id(s3tables):
"""A resent add-schema commit for an already-registered schema-id (Spark
re-declares its current schema on every write) must not append a second
entry -- Iceberg's own schemasById() crashes on load with "Multiple entries
with same key: <id>=table {...}" if the schemas list has a duplicate id."""
bucket_name = f"tb-schemadup-{_uuid_mod.uuid4().hex[:6]}"
bucket_arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
s3tables.create_namespace(tableBucketARN=bucket_arn, namespace=[ns])
s3tables.create_table(tableBucketARN=bucket_arn, namespace=ns, name=table, format="ICEBERG")
payload = {
"requirements": [],
"updates": [
{
"action": "add-schema",
"schema": {
"type": "struct", "schema-id": 0,
"fields": [{"id": 1, "name": "colA", "required": False, "type": "string"}],
},
},
{"action": "set-current-schema", "schema-id": 0},
],
}
# Send the identical commit twice, simulating Spark re-declaring its
# current schema on a second, unrelated write.
for _ in range(2):
_iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}", method="POST", payload=payload)
loaded = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")
schema_ids = [s.get("schema-id") for s in loaded["metadata"]["schemas"]]
assert schema_ids.count(0) == 1, "resent add-schema must not duplicate the schema-id entry"
finally:
try:
s3tables.delete_table(tableBucketARN=bucket_arn, namespace=ns, name=table)
except Exception:
pass
try:
s3tables.delete_namespace(tableBucketARN=bucket_arn, namespace=ns)
except Exception:
pass
s3tables.delete_table_bucket(tableBucketARN=bucket_arn)
def test_s3tables_iceberg_add_spec_is_idempotent_by_spec_id(s3tables):
"""A resent add-spec commit for an already-registered spec-id must not
append a second entry -- Iceberg's own specsById() crashes the same way
schemasById() does on a duplicate id."""
bucket_name = f"tb-specdup-{_uuid_mod.uuid4().hex[:6]}"
bucket_arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
s3tables.create_namespace(tableBucketARN=bucket_arn, namespace=[ns])
s3tables.create_table(tableBucketARN=bucket_arn, namespace=ns, name=table, format="ICEBERG")
payload = {
"requirements": [],
"updates": [
{
"action": "add-spec",
"spec": {
"spec-id": 1,
"fields": [{"source-id": 1, "field-id": 1000, "name": "day_col", "transform": "day"}],
},
},
{"action": "set-default-spec", "spec-id": 1},
],
}
for _ in range(2):
_iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}", method="POST", payload=payload)
loaded = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")
spec_ids = [s.get("spec-id") for s in loaded["metadata"]["partition-specs"]]
assert spec_ids.count(1) == 1, "resent add-spec must not duplicate the spec-id entry"
finally:
try:
s3tables.delete_table(tableBucketARN=bucket_arn, namespace=ns, name=table)
except Exception:
pass
try:
s3tables.delete_namespace(tableBucketARN=bucket_arn, namespace=ns)
except Exception:
pass
s3tables.delete_table_bucket(tableBucketARN=bucket_arn)
def test_s3tables_iceberg_add_sort_order_is_idempotent_by_order_id(s3tables):
"""A resent add-sort-order commit for an already-registered order-id must
not append a second entry -- Iceberg's own sortOrdersById() crashes the
same way schemasById() does on a duplicate id."""
bucket_name = f"tb-orderdup-{_uuid_mod.uuid4().hex[:6]}"
bucket_arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
ns = f"ns_{_uuid_mod.uuid4().hex[:6]}"
table = f"t_{_uuid_mod.uuid4().hex[:6]}"
try:
s3tables.create_namespace(tableBucketARN=bucket_arn, namespace=[ns])
s3tables.create_table(tableBucketARN=bucket_arn, namespace=ns, name=table, format="ICEBERG")
payload = {
"requirements": [],
"updates": [
{
"action": "add-sort-order",
"sort-order": {
"order-id": 1,
"fields": [{"source-id": 1, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}],
},
},
{"action": "set-default-sort-order", "sort-order-id": 1},
],
}
for _ in range(2):
_iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}", method="POST", payload=payload)
loaded = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")
order_ids = [o.get("order-id") for o in loaded["metadata"]["sort-orders"]]
assert order_ids.count(1) == 1, "resent add-sort-order must not duplicate the order-id entry"
finally:
try:
s3tables.delete_table(tableBucketARN=bucket_arn, namespace=ns, name=table)
except Exception:
pass
try:
s3tables.delete_namespace(tableBucketARN=bucket_arn, namespace=ns)
except Exception:
pass
s3tables.delete_table_bucket(tableBucketARN=bucket_arn)
def test_s3tables_deletes_answer_204(s3tables):
"""DeleteNamespace / DeleteTable / DeleteTableBucket answer 204 No Content."""
arn = s3tables.create_table_bucket(name="del-status-bkt")["arn"]
s3tables.create_namespace(tableBucketARN=arn, namespace=["ns204"])
s3tables.create_table(tableBucketARN=arn, namespace="ns204", name="t204",
format="ICEBERG")
r = s3tables.delete_table(tableBucketARN=arn, namespace="ns204", name="t204")
assert r["ResponseMetadata"]["HTTPStatusCode"] == 204
r = s3tables.delete_namespace(tableBucketARN=arn, namespace="ns204")
assert r["ResponseMetadata"]["HTTPStatusCode"] == 204
r = s3tables.delete_table_bucket(tableBucketARN=arn)
assert r["ResponseMetadata"]["HTTPStatusCode"] == 204
def test_s3tables_iceberg_load_table_serves_the_numerically_latest_metadata(s3tables, s3):
"""LoadTable's scan for the newest metadata.json must order vN numerically:
lexically v10 sorts before v2, so from the 10th commit on a lexical scan
serves v9 forever while the table keeps advancing."""
bucket_name = f"tb-vsort-{_uuid_mod.uuid4().hex[:6]}"
bucket_arn = s3tables.create_table_bucket(name=bucket_name)["arn"]
ns, table = "vsortns", "vsorttbl"
try:
s3tables.create_namespace(tableBucketARN=bucket_arn, namespace=[ns])
s3tables.create_table(tableBucketARN=bucket_arn, namespace=ns, name=table, format="ICEBERG")
base = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")["metadata"]
for v in range(1, 12):
doc = dict(base, **{"last-sequence-number": v})
s3.put_object(
Bucket=bucket_name,
Key=f"{ns}/{table}/metadata/v{v}.metadata.json",
Body=json.dumps(doc).encode(),
)
resp = _iceberg_json(f"/iceberg/v1/{_arn_prefix(bucket_arn)}/namespaces/{ns}/tables/{table}")
assert resp["metadata-location"].endswith("/v11.metadata.json"), resp["metadata-location"]
assert resp["metadata"]["last-sequence-number"] == 11
finally:
try:
s3tables.delete_table(tableBucketARN=bucket_arn, namespace=ns, name=table)
except Exception:
pass
try:
s3tables.delete_namespace(tableBucketARN=bucket_arn, namespace=ns)
except Exception:
pass
s3tables.delete_table_bucket(tableBucketARN=bucket_arn)
def test_s3tables_iceberg_prefixed_lookup_scopes_to_the_bucket(s3tables):
"""Two table buckets holding a same-named table must resolve per the
``{account}:s3tablescatalog/{bucket}`` prefix — without the filter the
first match wins and the wrong table's schema comes back."""
ns, table = "scopens", "orders"
arns = {}
try:
for b in ("tb-scope-a", "tb-scope-b"):
arns[b] = s3tables.create_table_bucket(name=b)["arn"]
s3tables.create_namespace(tableBucketARN=arns[b], namespace=[ns])
s3tables.create_table(
tableBucketARN=arns[b], namespace=ns, name=table, format="ICEBERG",
metadata={"iceberg": {"schema": {"fields": [
{"name": f"col_{b[-1]}", "type": "string"}]}}},
)
for b in ("tb-scope-a", "tb-scope-b"):
resp = _iceberg_json(
f"/iceberg/v1/000000000000:s3tablescatalog/{b}/namespaces/{ns}/tables/{table}")
fields = resp["metadata"]["schemas"][0]["fields"]
assert [f["name"] for f in fields] == [f"col_{b[-1]}"], (b, fields)
finally:
for b, arn in arns.items():
try:
s3tables.delete_table(tableBucketARN=arn, namespace=ns, name=table)
s3tables.delete_namespace(tableBucketARN=arn, namespace=ns)
except Exception:
pass
try:
s3tables.delete_table_bucket(tableBucketARN=arn)
except Exception:
pass
def test_iceberg_unknown_path_answers_404_envelope():
"""A path neither catalog serves must be an Iceberg error envelope, not an
empty 200 a client would read as success."""
import urllib.error
try:
_iceberg_json("/iceberg/v9/definitely/not/a/route")
raise AssertionError("expected HTTP error")
except urllib.error.HTTPError as e:
assert e.code == 404
doc = json.loads(e.read())
assert doc["error"]["type"] == "NotFoundException"
def _iceberg_call(path, method="GET", payload=None):
try:
return 200, _iceberg_json(path, method, payload)
except urllib.error.HTTPError as exc:
return exc.code, json.loads(exc.read().decode("utf-8") or "{}")
def _arn_prefix(arn):
return urllib.parse.quote(arn, safe="")
def test_s3tables_iceberg_rest_scopes_each_catalog_to_its_warehouse_arn(s3tables):
"""Two catalogs, each with its table-bucket ARN as warehouse; the REST
{prefix} "is always your url-encoded table bucket ARN"."""
suffix = _uuid_mod.uuid4().hex[:8]
arns = [s3tables.create_table_bucket(name=f"wh-{n}-{suffix}")["arn"] for n in ("std", "qual")]
try:
for arn in arns:
p = _arn_prefix(arn)
assert _iceberg_call(f"/iceberg/v1/{p}/namespaces", "POST", {"namespace": ["ns"]})[0] == 200
status, created = _iceberg_call(f"/iceberg/v1/{p}/namespaces/ns/tables", "POST", {
"name": "t", "schema": {"type": "struct", "schema-id": 0,
"fields": [{"id": 1, "name": "id", "type": "long", "required": False}]}})
assert status == 200, created
for arn in arns:
p = _arn_prefix(arn)
bucket = arn.rsplit("/", 1)[-1]
status, loaded = _iceberg_call(f"/iceberg/v1/{p}/namespaces/ns/tables/t")
assert status == 200
assert loaded["metadata-location"].startswith(f"s3://{bucket}/")
assert _iceberg_call(f"/iceberg/v1/{p}/namespaces/ns/tables")[1]["identifiers"] == [
{"namespace": ["ns"], "name": "t"}]
finally:
for arn in arns:
with contextlib.suppress(Exception):
s3tables.delete_table(tableBucketARN=arn, namespace="ns", name="t")
s3tables.delete_namespace(tableBucketARN=arn, namespace="ns")
s3tables.delete_table_bucket(tableBucketARN=arn)
def test_s3tables_iceberg_rest_sequence_numbers_and_requirements(s3tables):
arn = s3tables.create_table_bucket(name=f"seq-{_uuid_mod.uuid4().hex[:8]}")["arn"]
p = _arn_prefix(arn)
table = f"/iceberg/v1/{p}/namespaces/ns/tables/t"
try:
_iceberg_call(f"/iceberg/v1/{p}/namespaces", "POST", {"namespace": ["ns"]})
_iceberg_call(f"/iceberg/v1/{p}/namespaces/ns/tables", "POST", {
"name": "t", "format-version": 2, "schema": {"type": "struct", "schema-id": 0, "fields": [
{"id": 1, "name": "id", "type": "long", "required": False}]}})
def append(seq, snap_id, parent, expected_ref):
snapshot = {"snapshot-id": snap_id, "sequence-number": seq, "timestamp-ms": 1_700_000_000_000 + seq,
"manifest-list": f"s3://x/{snap_id}.avro", "summary": {"operation": "append"}}
if parent is not None:
snapshot["parent-snapshot-id"] = parent
return _iceberg_call(table, "POST", {
"requirements": [{"type": "assert-ref-snapshot-id", "ref": "main", "snapshot-id": expected_ref}],
"updates": [{"action": "add-snapshot", "snapshot": snapshot},
{"action": "set-snapshot-ref", "ref-name": "main", "type": "branch",
"snapshot-id": snap_id}]})
for seq in (1, 2, 3): # sequential Spark appends
status, body = append(seq, 100 + seq, (99 + seq) if seq > 1 else None, (99 + seq) if seq > 1 else None)
assert status == 200, body
assert body["metadata"]["last-sequence-number"] == seq
assert _iceberg_call(table)[1]["metadata"]["last-sequence-number"] == 3
# A writer that loaded the table before the last commit is refused.
status, body = append(3, 200, 102, 102)
assert status == 409
assert body["error"]["type"] == "CommitFailedException"
assert body["error"]["message"] == "Requirement failed: branch main has changed: expected id 102 != 103"
# A snapshot not newer than the table's last sequence number is refused.
status, body = _iceberg_call(table, "POST", {"updates": [{"action": "add-snapshot", "snapshot": {
"snapshot-id": 300, "sequence-number": 3, "parent-snapshot-id": 103, "timestamp-ms": 1_800_000_000_000,
"manifest-list": "s3://x/300.avro", "summary": {"operation": "append"}}}]})
assert status == 400
assert body["error"]["message"] == "Cannot add snapshot with sequence number 3 older than last sequence number 3"
finally:
with contextlib.suppress(Exception):
s3tables.delete_table(tableBucketARN=arn, namespace="ns", name="t")
s3tables.delete_namespace(tableBucketARN=arn, namespace="ns")
s3tables.delete_table_bucket(tableBucketARN=arn)
def test_iceberg_route_splits_paths_as_the_router():
"""The router and the IAM checks read one split: raw segments, the {prefix} decoded only afterwards."""
from ministack.services.s3tables import _iceberg_route, iceberg_lacks_prefix
bucket = "arn%3Aaws%3As3tables%3Aus-east-1%3A000000000000%3Abucket%2Fb"
assert _iceberg_route("GET", "/iceberg/v1/config") == ("config", [], [])
assert _iceberg_route("GET", f"/iceberg/v1/{bucket}/namespaces/ns") == ("namespaces", [bucket], ["ns"])
# a decoded ARN spans two segments; an encoded keyword is a segment like any other
assert _iceberg_route("GET", "/iceberg/v1/arn:aws:s3tables:us-east-1:000000000000:bucket/b/namespaces") == (
"namespaces", ["arn:aws:s3tables:us-east-1:000000000000:bucket", "b"], [])
assert _iceberg_route("GET", f"/iceberg/v1/%6Eamespaces/{bucket}/namespaces") == (
"namespaces", ["%6Eamespaces", bucket], [])
assert _iceberg_route("GET", f"/iceberg/v1/{bucket}/%6Eamespaces") is None
# tables/rename after a namespace is updateTable on a table named rename
assert _iceberg_route("POST", f"/iceberg/v1/{bucket}/tables/rename") == ("rename", [bucket], [])
assert _iceberg_route("POST", f"/iceberg/v1/{bucket}/namespaces/ns/tables/rename") == (
"namespaces", [bucket], ["ns", "tables", "rename"])
assert _iceberg_route("POST", f"/iceberg/v1/{bucket}/transactions/commit") == (
"transactions", [bucket], ["commit"])
for method, path in (("GET", "/iceberg/v1/namespaces"), ("GET", "/iceberg/v1/namespaces/ns/tables/t"),
("POST", "/iceberg/v1/tables/rename"), ("POST", "/iceberg/v1/transactions/commit")):
assert iceberg_lacks_prefix(method, path)
# a first segment before "namespaces" is the {prefix}, whatever it reads
for method, path in (("GET", "/iceberg/v1/config"), ("GET", f"/iceberg/v1/{bucket}/namespaces"),
("POST", f"/iceberg/v1/{bucket}/tables/rename"), ("GET", "/iceberg/v1/tables/namespaces"),
("GET", "/iceberg/v1/transactions/namespaces/ns"),
("POST", f"/iceberg/v1/{bucket}/transactions/commit")):
assert not iceberg_lacks_prefix(method, path)
def test_iceberg_iam_checks_name_the_bucket_the_prefix_names():
"""A bucket ARN or account:s3tablescatalog/bucket names a table bucket; any other {prefix} names none."""
from ministack.services.s3tables import iceberg_iam_checks, iceberg_prefix_error
bucket = "arn:aws:s3tables:us-east-1:000000000000:bucket/b"
for prefix in (_arn_prefix(bucket), "000000000000:s3tablescatalog/b"):
assert iceberg_iam_checks("GET", f"/iceberg/v1/{prefix}/namespaces", b"", {}) == [
("s3tables:ListNamespaces", bucket)]
assert iceberg_prefix_error("GET", f"/iceberg/v1/{prefix}/namespaces") is None
no_arn = "The specified table bucket ARN is not valid."
for prefix, message in (("catalog", no_arn), ("%6Eamespaces/000000000000:s3tablescatalog/b", no_arn),
(_arn_prefix(f"{bucket}/x"), "The specified bucket name isn't valid. Specify a different "
"bucket name, and then try again.")):
assert iceberg_iam_checks("POST", f"/iceberg/v1/{prefix}/namespaces", b"{}", {}) is None
status, _, body = iceberg_prefix_error("POST", f"/iceberg/v1/{prefix}/namespaces")
assert (status, json.loads(body)) == (400, {"error": {"message": message, "type": "bad_request", "code": 400}})
# a rename body whose source is no object names no table
assert iceberg_iam_checks("POST", f"/iceberg/v1/{_arn_prefix(bucket)}/tables/rename", b'{"source": "t"}', {}) == [
("s3tables:RenameTable", f"{bucket}/table/*")]
@pytest.fixture
def iceberg_auth_catalog(s3tables):
"""A table bucket with namespace ns and tables t and u, removed with whatever a test left in it."""
name = f"irc-{_uuid_mod.uuid4().hex[:8]}"
bucket = s3tables.create_table_bucket(name=name)["arn"]
s3tables.create_namespace(tableBucketARN=bucket, namespace=["ns"])
tables = {t: s3tables.create_table(tableBucketARN=bucket, namespace="ns", name=t, format="ICEBERG")["tableARN"]
for t in ("t", "u")}
yield {"bucket": bucket, "name": name, "prefix": _arn_prefix(bucket), **tables}
for namespace in s3tables.list_namespaces(tableBucketARN=bucket)["namespaces"]:
ns = namespace["namespace"][0]
for table in s3tables.list_tables(tableBucketARN=bucket, namespace=ns)["tables"]:
s3tables.delete_table(tableBucketARN=bucket, namespace=ns, name=table["name"])
s3tables.delete_namespace(tableBucketARN=bucket, namespace=ns)
s3tables.delete_table_bucket(tableBucketARN=bucket)
@contextlib.contextmanager
def _iceberg_contract_signer(user):
"""A second access key of an ``iam_user`` for signed Iceberg REST calls, deleted before the fixture's cleanup."""
from botocore.credentials import Credentials
from conftest import make_client
iam = make_client("iam")
key = iam.create_access_key(UserName=user.name)["AccessKey"]
try:
yield Credentials(key["AccessKeyId"], key["SecretAccessKey"])
finally:
iam.delete_access_key(UserName=user.name, AccessKeyId=key["AccessKeyId"])
def _iceberg_signed(credentials, method, path, query=None, body=None, service="s3tables", region="us-east-1",
presigned=False):
"""One SigV4-signed Iceberg REST call over HTTP: status, body (parsed when JSON, None when empty) and headers."""
from botocore.auth import S3SigV4Auth, SigV4Auth, SigV4QueryAuth
from botocore.awsrequest import AWSRequest
request = AWSRequest(method=method, url=f"{_ENDPOINT}/iceberg/v1/{path}", params=query or {},
data=json.dumps(body).encode() if body is not None else None)
if presigned:
SigV4QueryAuth(credentials, service, region, expires=300).add_auth(request)
else:
(S3SigV4Auth if service == "s3" else SigV4Auth)(credentials, service, region).add_auth(request)
prepared = request.prepare()
try:
with urllib.request.urlopen(urllib.request.Request(
prepared.url, data=prepared.body, method=method, headers=dict(prepared.headers))) as response:
status, content, headers = response.status, response.read(), response.headers
except urllib.error.HTTPError as error:
status, content, headers = error.code, error.read(), error.headers
headers = {k.lower(): v for k, v in headers.items()}
if content and "json" in headers.get("content-type", ""):
content = json.loads(content)
return status, content or None, headers
def _iceberg_fill(value, names):
"""A path, query or body template with {name} placeholders filled from ``names``."""
text = json.dumps(value)
for key, name in names.items():
text = text.replace("{" + key + "}", name)
return json.loads(text)
def _iceberg_denial(user, action, resource, reason=None):
"""The Iceberg error body of an s3tables identity-policy denial."""
reason = reason or f"because no identity-based policy allows the s3tables:{action} action"
return {"error": {"message": f"User: {user.arn} is not authorized to perform: s3tables:{action} "
f"on resource: {resource} {reason}", "type": "forbidden", "code": 403}}
# Iceberg REST routes on the S3 Tables catalog: method, path, query, body, the actions AWS lists for each (on the
# table bucket or table), and MiniStack's status once they are allowed.
_UPDATE_TABLE_T = [("UpdateTableMetadataLocation", "{t}"), ("PutTableData", "{t}"), ("GetTableData", "{t}")]
_S3TABLES_ICEBERG_ROUTES = {
"config": ("GET", "config", {"warehouse": "{bucket}"}, None, [("GetTableBucket", "{bucket}")], 200),
"list-namespaces": ("GET", "{prefix}/namespaces", None, None, [("ListNamespaces", "{bucket}")], 200),
"create-namespace": ("POST", "{prefix}/namespaces", None, {"namespace": ["{new}"]},
[("CreateNamespace", "{bucket}")], 200),
"load-namespace": ("GET", "{prefix}/namespaces/ns", None, None, [("GetNamespace", "{bucket}")], 200),
"drop-namespace": ("DELETE", "{prefix}/namespaces/ns", None, None, [("DeleteNamespace", "{bucket}")], 404),
"list-tables": ("GET", "{prefix}/namespaces/ns/tables", None, None, [("ListTables", "{bucket}")], 200),
"create-table": ("POST", "{prefix}/namespaces/ns/tables", None, {"name": "{new}"},
[("CreateTable", "{bucket}"), ("PutTableData", "{bucket}/table/*")], 200),
"load-table": ("GET", "{prefix}/namespaces/ns/tables/t", None, None,
[("GetTableMetadataLocation", "{t}"), ("GetTableData", "{t}")], 200),
"update-table": ("POST", "{prefix}/namespaces/ns/tables/t", None, {"requirements": [], "updates": []},
_UPDATE_TABLE_T, 200),
"drop-table": ("DELETE", "{prefix}/namespaces/ns/tables/t", None, None, [("DeleteTable", "{t}")], 404),
"table-exists": ("HEAD", "{prefix}/namespaces/ns/tables/t", None, None, [("GetTable", "{t}")], 200),
"rename-table": ("POST", "{prefix}/tables/rename", None, {"source": {"namespace": ["ns"], "name": "t"},
"destination": {"namespace": ["ns"], "name": "{new}"}},
[("RenameTable", "{t}")], 404),
"commit-transaction": ("POST", "{prefix}/transactions/commit", None, {"table-changes": [
{"identifier": {"namespace": ["ns"], "name": "t"}, "requirements": [], "updates": []}]}, _UPDATE_TABLE_T, 200),
}
@pytest.mark.auth_contract
@pytest.mark.parametrize("service", ["s3tables", "glue", "s3", "sts"])
@pytest.mark.parametrize("route", list(_S3TABLES_ICEBERG_ROUTES))
def test_s3tables_iceberg_route_contract(auth_enabled, iam_user, iceberg_auth_catalog, route, service):
"""Each Iceberg route on a table bucket needs exactly its s3tables actions, whatever service it is signed for."""
method, path, query, body, actions, served = _S3TABLES_ICEBERG_ROUTES[route]
actions = _iceberg_fill(actions, iceberg_auth_catalog)
allowed = iam_user(policy={"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": f"s3tables:{action}", "Resource": resource} for action, resource in actions]})
nobody = iam_user()
with _iceberg_contract_signer(allowed) as signer, _iceberg_contract_signer(nobody) as refused_signer:
calls = [_iceberg_signed(credentials, method, *_iceberg_fill([path, query, body], {
**iceberg_auth_catalog, "new": f"by-{label}"}), service=service)
for label, credentials in (("allowed", signer), ("nobody", refused_signer))]
assert calls[0][0] == served
if not auth_enabled:
assert calls[1][0] == served
return
status, error, headers = calls[1]
assert (status, headers["content-type"], headers["x-amzn-errortype"]) == (
403, "application/json", "IrcForbiddenResponse")
assert error == (None if method == "HEAD" else _iceberg_denial(nobody, *actions[0]))
@pytest.mark.auth_contract
@pytest.mark.parametrize("method,path,body,action,resource,reason", [
("GET", "config?warehouse={absent}", None, "GetTableBucket", "{absent}", None),
("POST", "{prefix}/namespaces", {"namespace": ["ns2"]}, "CreateNamespace", "{bucket}",
"with an explicit deny in an identity-based policy"),
("GET", "{prefix}/namespaces/ns/tables/u", None, "GetTableMetadataLocation", "{u}", None),
("GET", "{prefix}/namespaces/ns/tables/absent", None, "GetTableMetadataLocation", "{bucket}/table/*", None),
], ids=["config-absent-bucket", "explicit-deny", "other-table", "absent-table"])
def test_s3tables_iceberg_authorization_contract(auth_enabled, iam_user, iceberg_auth_catalog,
method, path, body, action, resource, reason):
"""A denial names the bucket or table the call names, table/* for a table that does not exist, and a Deny."""
catalog = iceberg_auth_catalog
user = iam_user(policy={"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Resource": catalog["bucket"],
"Action": ["s3tables:GetTableBucket", "s3tables:ListNamespaces", "s3tables:GetNamespace"]},
{"Effect": "Allow", "Action": "s3tables:GetTableMetadataLocation", "Resource": catalog["t"]},
{"Effect": "Deny", "Action": "s3tables:CreateNamespace", "Resource": catalog["bucket"]},
]})
names = {**catalog, "absent": catalog["bucket"].rsplit("/", 1)[0] + "/absent"}
path, query = _iceberg_fill(path, names), None
if "?warehouse=" in path:
path, warehouse = path.split("?warehouse=")
query = {"warehouse": warehouse}
with _iceberg_contract_signer(user) as signer:
assert _iceberg_signed(signer, "GET", "config", {"warehouse": catalog["bucket"]})[0] == 200
assert _iceberg_signed(signer, "GET", f"{catalog['prefix']}/namespaces")[1] == {"namespaces": [["ns"]]}
status, error, headers = _iceberg_signed(signer, method, path, query, body)
if not auth_enabled:
assert status != 403
return
assert (status, headers["content-type"]) == (403, "application/json")
assert error == _iceberg_denial(user, action, _iceberg_fill(resource, names), reason)
@pytest.mark.auth_contract
def test_s3tables_iceberg_later_action_contract(auth_enabled, iam_user, iceberg_auth_catalog):
"""loadTable's second action answers Access Denied."""
catalog = iceberg_auth_catalog
user = iam_user(policy={"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": "s3tables:GetTableMetadataLocation", "Resource": catalog["t"]}]})
with _iceberg_contract_signer(user) as signer:
loaded = _iceberg_signed(signer, "GET", f"{catalog['prefix']}/namespaces/ns/tables/t")
if not auth_enabled:
assert loaded[0] == 200
return
assert loaded[:2] == (403, {"error": {"message": "Access Denied", "type": "forbidden", "code": 403}})
@pytest.mark.auth_contract
@pytest.mark.parametrize("service", ["s3tables", "glue"])
def test_s3tables_iceberg_wrong_secret_contract(auth_enabled, iam_user, iceberg_auth_catalog, service):
"""Iceberg REST calls check their signature, for S3 Tables and for the Glue catalog alike, in JSON."""
from botocore.credentials import Credentials
catalog = iceberg_auth_catalog
warehouse = catalog["bucket"] if service == "s3tables" else "000000000000"
action = "s3tables:GetTableBucket" if service == "s3tables" else "glue:GetCatalog"
user = iam_user(policy={"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": action, "Resource": "*"}]})
with _iceberg_contract_signer(user) as signer:
assert _iceberg_signed(signer, "GET", "config", {"warehouse": warehouse}, service=service)[0] == 200
status, _, headers = _iceberg_signed(Credentials(signer.access_key, "wrong-secret"), "GET", "config",
{"warehouse": warehouse}, service=service)
if not auth_enabled:
assert status == 200
return
assert (status, headers["x-amzn-errortype"], headers["content-type"]) == (
403, "InvalidSignatureException", "application/json")
@pytest.mark.auth_contract
def test_s3tables_iceberg_calls_naming_no_table_bucket_contract(auth_enabled, iam_user):
"""An unprefixed route answers 404 whatever its signature; config without a table bucket answers 400."""
from botocore.credentials import Credentials
with _iceberg_contract_signer(iam_user()) as signer:
assert _iceberg_signed(signer, "GET", "config")[0] == 400
wrong = Credentials(signer.access_key, "wrong-secret")
assert _iceberg_signed(wrong, "GET", "namespaces")[0] == 404
assert _iceberg_signed(wrong, "HEAD", "namespaces/ns")[0] == 404
status, _, headers = _iceberg_signed(wrong, "GET", "config")
read_as_prefix = _iceberg_signed(wrong, "GET", "tables/namespaces", service="glue")
assert (status, headers.get("x-amzn-errortype")) == (
(403, "InvalidSignatureException") if auth_enabled else (400, None))
# "tables" before "namespaces" is the {prefix}, so that call is authenticated however it is signed
assert read_as_prefix[0] == (403 if auth_enabled else 200)
@pytest.mark.auth_contract
@pytest.mark.parametrize("method,prefix,message,served", [
("GET", "somewhere", "The specified table bucket ARN is not valid.", [200, 200, 200]),
("POST", "%256Eamespaces/000000000000:s3tablescatalog/{name}", "The specified table bucket ARN is not valid.",
[200, 409, 409]),
("POST", "{bucket}/x", "The specified bucket name isn't valid. Specify a different bucket name, and then try "
"again.", [404, 404, 404]),
], ids=["list-namespaces", "encoded-namespaces-segment", "bucket-arn-and-more"])
def test_s3tables_iceberg_prefix_naming_no_table_bucket_contract(auth_enabled, iam_user, iceberg_auth_catalog,
s3tables, method, prefix, message, served):
"""A {prefix} that is no table bucket answers 400 after the credential check, before any policy, as on AWS."""
from botocore.credentials import Credentials
catalog = iceberg_auth_catalog
prefix = _iceberg_fill(prefix, catalog)
prefix = _arn_prefix(prefix) if prefix.startswith("arn:") else prefix
everything = iam_user(policy={"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": "s3tables:*", "Resource": "*"}]})
body = {"namespace": ["created"]} if method == "POST" else None
with _iceberg_contract_signer(iam_user()) as nobody, _iceberg_contract_signer(everything) as signer:
calls = [_iceberg_signed(credentials, method, f"{prefix}/namespaces", body=body)
for credentials in (nobody, signer, Credentials(signer.access_key, "wrong-secret"))]
if not auth_enabled:
assert [status for status, _, _ in calls] == served
return
for status, error, _ in calls[:2]:
assert (status, error) == (400, {"error": {"message": message, "type": "bad_request", "code": 400}})
assert (calls[2][0], calls[2][2]["x-amzn-errortype"]) == (403, "InvalidSignatureException")
assert [n["namespace"] for n in s3tables.list_namespaces(tableBucketARN=catalog["bucket"])["namespaces"]] == [
["ns"]]
@pytest.mark.auth_contract
@pytest.mark.parametrize("prefix", ["{prefix}/%6Eamespaces", "000000000000:s3tablescatalog/{name}/namespaces"],
ids=["encoded-namespaces-keyword", "account-catalog-prefix"])
def test_s3tables_iceberg_prefix_forms_served_are_authorized_contract(auth_enabled, iam_user, iceberg_auth_catalog,
prefix):
"""A path MiniStack serves as listNamespaces is authorized as one, whichever form names the bucket."""
catalog = iceberg_auth_catalog
path = _iceberg_fill(prefix, catalog)
allowed = iam_user(policy={"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": "s3tables:ListNamespaces", "Resource": catalog["bucket"]}]})
nobody = iam_user()
with _iceberg_contract_signer(allowed) as signer, _iceberg_contract_signer(nobody) as refused_signer:
listed = _iceberg_signed(signer, "GET", path)
refused = _iceberg_signed(refused_signer, "GET", path)
assert listed[:2] == (200, {"namespaces": [["ns"]]})
if not auth_enabled:
assert refused[:2] == listed[:2]
return
assert refused[:2] == (403, _iceberg_denial(nobody, "ListNamespaces", catalog["bucket"]))
@pytest.mark.auth_contract
def test_s3tables_iceberg_presigned_region_contract(auth_enabled, iam_user, iceberg_auth_catalog):
"""Under AUTH a presigned call serves its credential region, whose bucket and table the checks read."""
catalog = iceberg_auth_catalog
west = _make_s3tables_client(region_name="us-west-2")
west_arn = west.create_table_bucket(name=catalog["name"])["arn"]
west.create_namespace(tableBucketARN=west_arn, namespace=["ns"])
secret = west.create_table(tableBucketARN=west_arn, namespace="ns", name="secret", format="ICEBERG")["tableARN"]
reads = ["s3tables:GetTableMetadataLocation", "s3tables:GetTableData"]
user = iam_user(policy={"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": "s3tables:ListNamespaces", "Resource": catalog["bucket"]},
{"Effect": "Allow", "Action": reads, "Resource": "*"},
{"Effect": "Deny", "Action": reads, "Resource": secret}]})
by_name = f"000000000000:s3tablescatalog/{catalog['name']}/namespaces"
table = f"{_arn_prefix(west_arn)}/namespaces/ns/tables/secret"
try:
with _iceberg_contract_signer(user) as signer:
calls = {(region, kind): _iceberg_signed(signer, "GET", path, region=region, presigned=True)
for region in ("us-east-1", "us-west-2") for kind, path in (("list", by_name), ("table", table))}
finally:
west.delete_table(tableBucketARN=west_arn, namespace="ns", name="secret")
west.delete_namespace(tableBucketARN=west_arn, namespace="ns")
west.delete_table_bucket(tableBucketARN=west_arn)
if not auth_enabled:
assert [status for status, _, _ in calls.values()] == [200, 200, 200, 200]
return
# signed for us-east-1, neither call reaches the us-west-2 bucket
assert calls["us-east-1", "list"][:2] == (200, {"namespaces": [["ns"]]})
assert calls["us-east-1", "table"][0] == 404
assert calls["us-west-2", "list"][:2] == (403, _iceberg_denial(user, "ListNamespaces", west_arn))
assert calls["us-west-2", "table"][:2] == (403, _iceberg_denial(
user, "GetTableMetadataLocation", secret, "with an explicit deny in an identity-based policy"))
@pytest.mark.auth_contract
@pytest.mark.parametrize("warehouse", ["1:s3tablescatalog/{name}", "000000000000:s3tablescatalog/{name}/x"],
ids=["short-account", "trailing-segment"])
def test_s3tables_iceberg_config_catalog_warehouse_contract(auth_enabled, iam_user, iceberg_auth_catalog, warehouse):
"""getConfig serves any account:s3tablescatalog/ warehouse; one naming no bucket needs GetTableBucket on *."""
warehouse = _iceberg_fill(warehouse, iceberg_auth_catalog)
nobody = iam_user()
allowed = iam_user(policy={"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": "s3tables:GetTableBucket", "Resource": "*"}]})
with _iceberg_contract_signer(allowed) as signer, _iceberg_contract_signer(nobody) as refused_signer:
served = _iceberg_signed(signer, "GET", "config", {"warehouse": warehouse})
refused = _iceberg_signed(refused_signer, "GET", "config", {"warehouse": warehouse})
assert served[0] == 200
if not auth_enabled:
assert refused[0] == 200
return
assert refused[:2] == (403, _iceberg_denial(nobody, "GetTableBucket", "*"))
@pytest.mark.auth_contract
def test_s3tables_iceberg_table_named_rename_contract(auth_enabled, iam_user, iceberg_auth_catalog, s3tables):
"""A commit to a table named rename is updateTable on that table, not renameTable."""
catalog = iceberg_auth_catalog
table = s3tables.create_table(tableBucketARN=catalog["bucket"], namespace="ns", name="rename",
format="ICEBERG")["tableARN"]
path = f"{catalog['prefix']}/namespaces/ns/tables/rename"
renamer = iam_user(policy={"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": "s3tables:RenameTable", "Resource": "*"}]})
updater = iam_user(policy={"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": f"s3tables:{action}", "Resource": table} for action, _ in _UPDATE_TABLE_T]})
commit = {"requirements": [], "updates": []}
with _iceberg_contract_signer(renamer) as refused_signer, _iceberg_contract_signer(updater) as signer:
assert _iceberg_signed(signer, "POST", path, body=commit)[0] == 200
refused = _iceberg_signed(refused_signer, "POST", path, body=commit)
if not auth_enabled:
assert refused[0] == 200
return
assert refused[:2] == (403, _iceberg_denial(renamer, "UpdateTableMetadataLocation", table))
@pytest.mark.auth_contract
@pytest.mark.parametrize("body", ["t", "t-and-u", "no-changes", "no-object"])
def test_s3tables_iceberg_transaction_commit_contract(auth_enabled, iam_user, iceberg_auth_catalog, body):
"""transactions/commit needs updateTable's actions on each table it changes, on * when it names none."""
catalog = iceberg_auth_catalog
change = {"identifier": {"namespace": ["ns"], "name": "t"}, "requirements": [], "updates": []}
bodies = {"t": {"table-changes": [change]},
"t-and-u": {"table-changes": [change, {**change, "identifier": {"namespace": ["ns"], "name": "u"}}]},
"no-changes": {}, "no-object": "table-changes"}
allowed = iam_user(policy={"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": f"s3tables:{action}", "Resource": catalog["t"]}
for action, _ in _UPDATE_TABLE_T]})
nobody = iam_user()
path = f"{catalog['prefix']}/transactions/commit"
with _iceberg_contract_signer(allowed) as signer, _iceberg_contract_signer(nobody) as refused_signer:
committed = _iceberg_signed(signer, "POST", path, body=bodies[body])
refused = _iceberg_signed(refused_signer, "POST", path, body=bodies[body])
if not auth_enabled:
assert (committed[0], refused[0]) == ((500, 500) if body == "no-object" else (200, 200))
return
first = "*" if body in ("no-changes", "no-object") else catalog["t"]
assert refused[:2] == (403, _iceberg_denial(nobody, "UpdateTableMetadataLocation", first))
if body == "t":
assert committed[0] == 200
elif body == "t-and-u":
assert committed[:2] == (403, {"error": {"message": "Access Denied", "type": "forbidden", "code": 403}})
else:
assert committed[:2] == (403, _iceberg_denial(allowed, "UpdateTableMetadataLocation", "*"))
def test_s3tables_bucket_subresources_defaults_and_updates(s3tables):
"""Documented defaults until set: SSE-S3, STANDARD, unreferenced file removal 3/10 days."""
arn = s3tables.create_table_bucket(name=f"sub-{_uuid_mod.uuid4().hex[:8]}",
tags={"team": "data"})["arn"]
got = s3tables.get_table_bucket(tableBucketARN=arn)
assert set(got) - {"ResponseMetadata"} <= {"arn", "name", "ownerAccountId", "createdAt", "tableBucketId", "type"}
assert got["type"] == "customer" and got["tableBucketId"]
assert s3tables.get_table_bucket_encryption(tableBucketARN=arn)["encryptionConfiguration"] == {
"sseAlgorithm": "AES256"}
s3tables.put_table_bucket_encryption(tableBucketARN=arn, encryptionConfiguration={
"sseAlgorithm": "aws:kms", "kmsKeyArn": "arn:aws:kms:us-east-1:000000000000:key/k"})
assert s3tables.get_table_bucket_encryption(tableBucketARN=arn)["encryptionConfiguration"]["sseAlgorithm"] == "aws:kms"
s3tables.delete_table_bucket_encryption(tableBucketARN=arn)
assert s3tables.get_table_bucket_encryption(tableBucketARN=arn)["encryptionConfiguration"] == {
"sseAlgorithm": "AES256"}
removal = s3tables.get_table_bucket_maintenance_configuration(tableBucketARN=arn)["configuration"][
"icebergUnreferencedFileRemoval"]
assert removal == {"status": "enabled", "settings": {"icebergUnreferencedFileRemoval": {
"unreferencedDays": 3, "nonCurrentDays": 10}}}
s3tables.put_table_bucket_maintenance_configuration(
tableBucketARN=arn, type="icebergUnreferencedFileRemoval",
value={"status": "enabled", "settings": {"icebergUnreferencedFileRemoval": {"unreferencedDays": 4}}})
removal = s3tables.get_table_bucket_maintenance_configuration(tableBucketARN=arn)["configuration"][
"icebergUnreferencedFileRemoval"]
assert removal["settings"]["icebergUnreferencedFileRemoval"] == {"unreferencedDays": 4, "nonCurrentDays": 10}
with pytest.raises(ClientError) as exc:
s3tables.get_table_bucket_policy(tableBucketARN=arn)
assert exc.value.response["Error"]["Code"] == "NotFoundException"
s3tables.put_table_bucket_policy(tableBucketARN=arn, resourcePolicy='{"Version":"2012-10-17","Statement":[]}')
assert s3tables.get_table_bucket_policy(tableBucketARN=arn)["resourcePolicy"].startswith('{"Version"')
s3tables.delete_table_bucket_policy(tableBucketARN=arn)
assert s3tables.get_table_bucket_storage_class(tableBucketARN=arn)["storageClassConfiguration"] == {
"storageClass": "STANDARD"}
s3tables.put_table_bucket_storage_class(tableBucketARN=arn,
storageClassConfiguration={"storageClass": "INTELLIGENT_TIERING"})
assert s3tables.get_table_bucket_storage_class(tableBucketARN=arn)["storageClassConfiguration"][
"storageClass"] == "INTELLIGENT_TIERING"
s3tables.put_table_bucket_metrics_configuration(tableBucketARN=arn)
assert s3tables.get_table_bucket_metrics_configuration(tableBucketARN=arn)["id"]
s3tables.delete_table_bucket_metrics_configuration(tableBucketARN=arn)
assert s3tables.list_tags_for_resource(resourceArn=arn)["tags"] == {"team": "data"}
s3tables.tag_resource(resourceArn=arn, tags={"env": "dev"})
s3tables.untag_resource(resourceArn=arn, tagKeys=["team"])
assert s3tables.list_tags_for_resource(resourceArn=arn)["tags"] == {"env": "dev"}
def test_s3tables_table_subresources_and_rename(s3tables):
arn = s3tables.create_table_bucket(name=f"tsub-{_uuid_mod.uuid4().hex[:8]}")["arn"]
for ns in ("a", "b"):
s3tables.create_namespace(tableBucketARN=arn, namespace=[ns])
created = s3tables.create_table(tableBucketARN=arn, namespace="a", name="t", format="ICEBERG",
tags={"k": "v"})
table = s3tables.get_table(tableBucketARN=arn, namespace="a", name="t")
assert table["type"] == "customer" and table["versionToken"] == created["versionToken"]
assert "tableBucketARN" not in table
config = s3tables.get_table_maintenance_configuration(tableBucketARN=arn, namespace="a", name="t")["configuration"]
assert config["icebergCompaction"] == {"status": "enabled", "settings": {"icebergCompaction": {
"targetFileSizeMB": 512, "strategy": "auto"}}}
assert config["icebergSnapshotManagement"] == {"status": "enabled", "settings": {"icebergSnapshotManagement": {
"minSnapshotsToKeep": 1, "maxSnapshotAgeHours": 120}}}
s3tables.put_table_maintenance_configuration(
tableBucketARN=arn, namespace="a", name="t", type="icebergCompaction",
value={"status": "disabled", "settings": {"icebergCompaction": {"targetFileSizeMB": 256}}})
compaction = s3tables.get_table_maintenance_configuration(
tableBucketARN=arn, namespace="a", name="t")["configuration"]["icebergCompaction"]
assert compaction["status"] == "disabled"
assert compaction["settings"]["icebergCompaction"]["targetFileSizeMB"] == 256
status = s3tables.get_table_maintenance_job_status(tableBucketARN=arn, namespace="a", name="t")["status"]
assert {v["status"] for v in status.values()} == {"Not_Yet_Run"}
assert s3tables.get_table_encryption(tableBucketARN=arn, namespace="a", name="t")[
"encryptionConfiguration"] == {"sseAlgorithm": "AES256"}
assert s3tables.get_table_storage_class(tableBucketARN=arn, namespace="a", name="t")[
"storageClassConfiguration"] == {"storageClass": "STANDARD"}
with pytest.raises(ClientError) as exc:
s3tables.get_table_policy(tableBucketARN=arn, namespace="a", name="t")
assert exc.value.response["Error"]["Code"] == "NotFoundException"
s3tables.put_table_policy(tableBucketARN=arn, namespace="a", name="t", resourcePolicy="{}")
assert s3tables.get_table_policy(tableBucketARN=arn, namespace="a", name="t")["resourcePolicy"] == "{}"
assert s3tables.list_tags_for_resource(resourceArn=created["tableARN"])["tags"] == {"k": "v"}
s3tables.rename_table(tableBucketARN=arn, namespace="a", name="t", newNamespaceName="b", newName="t2")
renamed = s3tables.get_table(tableBucketARN=arn, namespace="b", name="t2")
assert renamed["tableARN"] == created["tableARN"] and renamed["namespace"] == ["b"]
with pytest.raises(ClientError) as exc:
s3tables.get_table(tableBucketARN=arn, namespace="a", name="t")
assert exc.value.response["Error"]["Code"] == "NotFoundException"
def test_s3tables_create_table_keeps_iceberg_types(s3tables):
bucket = s3tables.create_table_bucket(name=f"types-{_uuid_mod.uuid4().hex[:8]}")["arn"]
s3tables.create_namespace(tableBucketARN=bucket, namespace=["ns"])
primitives = ["decimal(19, 2)", "decimal(19,2)", "timestamp", "timestamptz", "time", "uuid", "fixed[16]", "binary",
"STRING"]
stored = ["decimal(19, 2)", "decimal(19, 2)", "timestamp", "timestamptz", "time", "uuid", "fixed[16]", "binary",
"string"]
nested = {"type": "list", "element-id": 3, "element": "string", "element-required": False}
try:
s3tables.create_table(
tableBucketARN=bucket, namespace="ns", name="flat", format="ICEBERG",
metadata={"iceberg": {"schema": {"fields": [
{"name": f"c{i}", "type": t, "required": False} for i, t in enumerate(primitives)]}}},
)
s3tables.create_table(
tableBucketARN=bucket, namespace="ns", name="nested", format="ICEBERG",
metadata={"iceberg": {"schemaV2": {"type": "struct", "fields": [
{"id": 1, "name": "amount", "type": "decimal(9,2)", "required": True},
{"id": 2, "name": "tags", "type": nested, "required": False}]}}},
)
prefix = _arn_prefix(bucket)
flat = _iceberg_json(f"/iceberg/v1/{prefix}/namespaces/ns/tables/flat")["metadata"]
assert [f["type"] for f in flat["schemas"][0]["fields"]] == stored
fields = _iceberg_json(f"/iceberg/v1/{prefix}/namespaces/ns/tables/nested")["metadata"]["schemas"][0]["fields"]
assert [f["type"] for f in fields] == ["decimal(9, 2)", nested]
finally:
for name in ("flat", "nested"):
with contextlib.suppress(ClientError):
s3tables.delete_table(tableBucketARN=bucket, namespace="ns", name=name)
s3tables.delete_namespace(tableBucketARN=bucket, namespace="ns")
s3tables.delete_table_bucket(tableBucketARN=bucket)
@pytest.mark.parametrize("kind", ["timestamp_ns", "varchar", "decimal(x,2)"])
def test_s3tables_create_table_refuses_an_invalid_iceberg_type(s3tables, kind):
bucket = s3tables.create_table_bucket(name=f"badtype-{_uuid_mod.uuid4().hex[:8]}")["arn"]
s3tables.create_namespace(tableBucketARN=bucket, namespace=["ns"])
try:
with pytest.raises(ClientError) as exc:
s3tables.create_table(
tableBucketARN=bucket, namespace="ns", name="t", format="ICEBERG",
metadata={"iceberg": {"schema": {"fields": [{"name": "c", "type": kind}]}}},
)
error = exc.value.response
assert error["ResponseMetadata"]["HTTPStatusCode"] == 400
assert (error["Error"]["Code"], error["Error"]["Message"]) == (
"BadRequestException", "The specified metadata is not valid")
assert s3tables.list_tables(tableBucketARN=bucket)["tables"] == []
finally:
s3tables.delete_namespace(tableBucketARN=bucket, namespace="ns")
s3tables.delete_table_bucket(tableBucketARN=bucket)