Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

45.7 KB
Raw Blame
import base64
import io
import json
import os
import re
import time
import uuid as _uuid_mod
import zipfile
from urllib.parse import urlparse

import boto3
import pytest
from botocore.config import Config
from botocore.exceptions import ClientError

ENDPOINT = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566")


def _regional_client(service: str, region: str):
    return boto3.client(
        service,
        endpoint_url=ENDPOINT,
        aws_access_key_id="test",
        aws_secret_access_key="test",
        region_name=region,
        config=Config(region_name=region, retries={"mode": "standard"}),
    )


def test_firehose_create_and_describe(fh):
    name = "intg-fh-basic"
    arn = fh.create_delivery_stream(
        DeliveryStreamName=name,
        DeliveryStreamType="DirectPut",
        ExtendedS3DestinationConfiguration={
            "BucketARN": "arn:aws:s3:::my-bucket",
            "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
        },
    )["DeliveryStreamARN"]
    assert "firehose" in arn
    assert name in arn

    desc = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    assert desc["DeliveryStreamName"] == name
    assert desc["DeliveryStreamStatus"] == "ACTIVE"
    assert desc["DeliveryStreamType"] == "DirectPut"
    assert len(desc["Destinations"]) == 1
    assert "ExtendedS3DestinationDescription" in desc["Destinations"][0]
    assert desc["VersionId"] == "1"


@pytest.mark.parametrize(
    "config_key",
    ("ExtendedS3DestinationConfiguration", "S3DestinationConfiguration"),
)
@pytest.mark.parametrize(
    "bucket_arn",
    (
        "not-an-arn",
        "arn:aws:sns:us-east-1:000000000000:topic-name",
        "arn:aws:s3:us-east-1::my-bucket",
        "arn:aws:s3::000000000000:my-bucket",
        "arn:aws:s3:::my-bucket/path/to/object",
        "arn:aws:s3:::my-bucket:extra",
        "arn:aws:s3:::bad*bucket",
    ),
)
def test_firehose_rejects_invalid_s3_bucket_arns(fh, config_key, bucket_arn):
    with pytest.raises(ClientError) as exc:
        fh.create_delivery_stream(
            DeliveryStreamName=f"intg-fh-bad-bucket-arn-{_uuid_mod.uuid4().hex[:8]}",
            DeliveryStreamType="DirectPut",
            **{
                config_key: {
                    "BucketARN": bucket_arn,
                    "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
                },
            },
        )
    assert exc.value.response["Error"]["Code"] == "InvalidArgumentException"


def test_firehose_list_streams(fh):
    fh.create_delivery_stream(DeliveryStreamName="intg-fh-list-a", DeliveryStreamType="DirectPut")
    fh.create_delivery_stream(DeliveryStreamName="intg-fh-list-b", DeliveryStreamType="DirectPut")
    resp = fh.list_delivery_streams()
    names = resp["DeliveryStreamNames"]
    assert "intg-fh-list-a" in names
    assert "intg-fh-list-b" in names
    assert resp["HasMoreDeliveryStreams"] is False

def test_firehose_put_record(fh):
    name = "intg-fh-put"
    fh.create_delivery_stream(DeliveryStreamName=name, DeliveryStreamType="DirectPut")
    import base64

    data = base64.b64encode(b"hello firehose").decode()
    resp = fh.put_record(DeliveryStreamName=name, Record={"Data": data})
    assert "RecordId" in resp
    assert len(resp["RecordId"]) > 0
    assert resp["Encrypted"] is False

def test_firehose_put_record_batch(fh):
    name = "intg-fh-batch"
    fh.create_delivery_stream(DeliveryStreamName=name, DeliveryStreamType="DirectPut")
    import base64

    records = [{"Data": base64.b64encode(f"record-{i}".encode()).decode()} for i in range(5)]
    resp = fh.put_record_batch(DeliveryStreamName=name, Records=records)
    assert resp["FailedPutCount"] == 0
    assert len(resp["RequestResponses"]) == 5
    for r in resp["RequestResponses"]:
        assert "RecordId" in r

def test_firehose_delete_stream(fh):
    name = "intg-fh-delete"
    fh.create_delivery_stream(DeliveryStreamName=name, DeliveryStreamType="DirectPut")
    fh.delete_delivery_stream(DeliveryStreamName=name)
    from botocore.exceptions import ClientError

    try:
        fh.describe_delivery_stream(DeliveryStreamName=name)
        assert False, "should have raised"
    except ClientError as e:
        assert e.response["Error"]["Code"] == "ResourceNotFoundException"

def test_firehose_tags(fh):
    name = "intg-fh-tags"
    fh.create_delivery_stream(DeliveryStreamName=name, DeliveryStreamType="DirectPut")
    fh.tag_delivery_stream(
        DeliveryStreamName=name,
        Tags=[
            {"Key": "Env", "Value": "test"},
            {"Key": "Team", "Value": "data"},
        ],
    )
    resp = fh.list_tags_for_delivery_stream(DeliveryStreamName=name)
    tag_map = {t["Key"]: t["Value"] for t in resp["Tags"]}
    assert tag_map["Env"] == "test"
    assert tag_map["Team"] == "data"

    fh.untag_delivery_stream(DeliveryStreamName=name, TagKeys=["Env"])
    resp2 = fh.list_tags_for_delivery_stream(DeliveryStreamName=name)
    keys = [t["Key"] for t in resp2["Tags"]]
    assert "Env" not in keys
    assert "Team" in keys

def test_firehose_update_destination(fh):
    name = "intg-fh-update-dest"
    fh.create_delivery_stream(
        DeliveryStreamName=name,
        DeliveryStreamType="DirectPut",
        ExtendedS3DestinationConfiguration={
            "BucketARN": "arn:aws:s3:::original-bucket",
            "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
        },
    )
    desc = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    dest_id = desc["Destinations"][0]["DestinationId"]
    version_id = desc["VersionId"]

    fh.update_destination(
        DeliveryStreamName=name,
        DestinationId=dest_id,
        CurrentDeliveryStreamVersionId=version_id,
        ExtendedS3DestinationUpdate={
            "BucketARN": "arn:aws:s3:::updated-bucket",
            "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
        },
    )
    desc2 = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    assert desc2["VersionId"] == "2"
    s3_cfg = desc2["Destinations"][0]["ExtendedS3DestinationDescription"]
    assert s3_cfg["BucketARN"] == "arn:aws:s3:::updated-bucket"

def test_firehose_encryption(fh):
    name = "intg-fh-enc"
    fh.create_delivery_stream(DeliveryStreamName=name, DeliveryStreamType="DirectPut")
    fh.start_delivery_stream_encryption(
        DeliveryStreamName=name,
        DeliveryStreamEncryptionConfigurationInput={"KeyType": "AWS_OWNED_CMK"},
    )
    desc = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    assert desc["DeliveryStreamEncryptionConfiguration"]["Status"] == "ENABLED"

    fh.stop_delivery_stream_encryption(DeliveryStreamName=name)
    desc2 = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    assert desc2["DeliveryStreamEncryptionConfiguration"]["Status"] == "DISABLED"

def test_firehose_duplicate_create_error(fh):
    name = "intg-fh-dup"
    fh.create_delivery_stream(DeliveryStreamName=name, DeliveryStreamType="DirectPut")
    from botocore.exceptions import ClientError

    try:
        fh.create_delivery_stream(DeliveryStreamName=name, DeliveryStreamType="DirectPut")
        assert False, "should have raised"
    except ClientError as e:
        assert e.response["Error"]["Code"] == "ResourceInUseException"

def test_firehose_not_found_error(fh):
    from botocore.exceptions import ClientError

    try:
        fh.describe_delivery_stream(DeliveryStreamName="no-such-stream-xyz")
        assert False, "should have raised"
    except ClientError as e:
        assert e.response["Error"]["Code"] == "ResourceNotFoundException"

def test_firehose_list_with_type_filter(fh):
    fh.create_delivery_stream(DeliveryStreamName="intg-fh-type-dp", DeliveryStreamType="DirectPut")
    resp = fh.list_delivery_streams(DeliveryStreamType="DirectPut")
    assert "intg-fh-type-dp" in resp["DeliveryStreamNames"]

def test_firehose_s3_dest_has_encryption_config(fh):
    name = "intg-fh-enc-cfg"
    fh.create_delivery_stream(
        DeliveryStreamName=name,
        DeliveryStreamType="DirectPut",
        ExtendedS3DestinationConfiguration={
            "BucketARN": "arn:aws:s3:::my-bucket",
            "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
        },
    )
    desc = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    s3 = desc["Destinations"][0]["ExtendedS3DestinationDescription"]
    assert "EncryptionConfiguration" in s3
    assert s3["EncryptionConfiguration"] == {"NoEncryptionConfig": "NoEncryption"}

def test_firehose_no_enc_config_when_not_set(fh):
    name = "intg-fh-no-enc"
    fh.create_delivery_stream(DeliveryStreamName=name, DeliveryStreamType="DirectPut")
    desc = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    assert "DeliveryStreamEncryptionConfiguration" not in desc

def test_firehose_kinesis_source_block(fh):
    name = "intg-fh-kinesis-src"
    fh.create_delivery_stream(
        DeliveryStreamName=name,
        DeliveryStreamType="KinesisStreamAsSource",
        KinesisStreamSourceConfiguration={
            "KinesisStreamARN": "arn:aws:kinesis:us-east-1:000000000000:stream/my-stream",
            "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
        },
        ExtendedS3DestinationConfiguration={
            "BucketARN": "arn:aws:s3:::my-bucket",
            "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
        },
    )
    desc = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    assert "Source" in desc
    ks = desc["Source"]["KinesisStreamSourceDescription"]
    assert ks["KinesisStreamARN"] == "arn:aws:kinesis:us-east-1:000000000000:stream/my-stream"
    assert ks["RoleARN"] == "arn:aws:iam::000000000000:role/firehose-role"
    assert "DeliveryStartTimestamp" in ks

def test_firehose_update_destination_merges_same_type(fh):
    name = "intg-fh-merge"
    fh.create_delivery_stream(
        DeliveryStreamName=name,
        DeliveryStreamType="DirectPut",
        ExtendedS3DestinationConfiguration={
            "BucketARN": "arn:aws:s3:::original-bucket",
            "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
            "Prefix": "original/",
        },
    )
    desc = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    dest_id = desc["Destinations"][0]["DestinationId"]

    fh.update_destination(
        DeliveryStreamName=name,
        DestinationId=dest_id,
        CurrentDeliveryStreamVersionId=desc["VersionId"],
        ExtendedS3DestinationUpdate={
            "BucketARN": "arn:aws:s3:::updated-bucket",
        },
    )
    desc2 = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    s3 = desc2["Destinations"][0]["ExtendedS3DestinationDescription"]
    # Updated field
    assert s3["BucketARN"] == "arn:aws:s3:::updated-bucket"
    # Merged field preserved
    assert s3["Prefix"] == "original/"
    assert s3["RoleARN"] == "arn:aws:iam::000000000000:role/firehose-role"

def test_firehose_update_destination_replaces_on_type_change(fh):
    name = "intg-fh-type-change"
    fh.create_delivery_stream(
        DeliveryStreamName=name,
        DeliveryStreamType="DirectPut",
        ExtendedS3DestinationConfiguration={
            "BucketARN": "arn:aws:s3:::my-bucket",
            "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
        },
    )
    desc = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    dest_id = desc["Destinations"][0]["DestinationId"]

    fh.update_destination(
        DeliveryStreamName=name,
        DestinationId=dest_id,
        CurrentDeliveryStreamVersionId=desc["VersionId"],
        HttpEndpointDestinationUpdate={
            "EndpointConfiguration": {"Url": "https://my-endpoint.example.com"},
        },
    )
    desc2 = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    dest = desc2["Destinations"][0]
    assert "HttpEndpointDestinationDescription" in dest
    assert "ExtendedS3DestinationDescription" not in dest

def test_firehose_put_record_batch_failure_count(fh):
    """PutRecordBatch with valid records returns FailedPutCount=0."""
    fh.create_delivery_stream(
        DeliveryStreamName="qa-fh-batch-fail",
        ExtendedS3DestinationConfiguration={
            "BucketARN": "arn:aws:s3:::qa-fh-bucket",
            "RoleARN": "arn:aws:iam::000000000000:role/r",
        },
    )
    resp = fh.put_record_batch(
        DeliveryStreamName="qa-fh-batch-fail",
        Records=[{"Data": "aGVsbG8="}, {"Data": "d29ybGQ="}],
    )
    assert resp["FailedPutCount"] == 0
    assert len(resp["RequestResponses"]) == 2

def test_firehose_update_destination_version_mismatch(fh):
    """UpdateDestination with wrong version raises ConcurrentModificationException."""
    fh.create_delivery_stream(
        DeliveryStreamName="qa-fh-version-check",
        ExtendedS3DestinationConfiguration={
            "BucketARN": "arn:aws:s3:::qa-fh-bucket2",
            "RoleARN": "arn:aws:iam::000000000000:role/r",
        },
    )
    desc = fh.describe_delivery_stream(DeliveryStreamName="qa-fh-version-check")
    dest_id = desc["DeliveryStreamDescription"]["Destinations"][0]["DestinationId"]
    with pytest.raises(ClientError) as exc:
        fh.update_destination(
            DeliveryStreamName="qa-fh-version-check",
            CurrentDeliveryStreamVersionId="999",
            DestinationId=dest_id,
            ExtendedS3DestinationUpdate={
                "BucketARN": "arn:aws:s3:::qa-fh-bucket2-updated",
                "RoleARN": "arn:aws:iam::000000000000:role/r",
            },
        )
    assert exc.value.response["Error"]["Code"] == "ConcurrentModificationException"


def test_firehose_update_destination_rejects_invalid_s3_bucket_arn(fh):
    name = f"qa-fh-update-bad-bucket-arn-{_uuid_mod.uuid4().hex[:8]}"
    fh.create_delivery_stream(
        DeliveryStreamName=name,
        ExtendedS3DestinationConfiguration={
            "BucketARN": "arn:aws:s3:::qa-fh-bucket3",
            "RoleARN": "arn:aws:iam::000000000000:role/r",
        },
    )
    desc = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    dest_id = desc["Destinations"][0]["DestinationId"]
    version_id = desc["VersionId"]

    with pytest.raises(ClientError) as exc:
        fh.update_destination(
            DeliveryStreamName=name,
            CurrentDeliveryStreamVersionId=version_id,
            DestinationId=dest_id,
            ExtendedS3DestinationUpdate={
                "BucketARN": "arn:aws:s3:::qa-fh-bucket3/object",
            },
        )

    assert exc.value.response["Error"]["Code"] == "InvalidArgumentException"
    after = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    s3 = after["Destinations"][0]["ExtendedS3DestinationDescription"]
    assert after["VersionId"] == version_id
    assert s3["BucketARN"] == "arn:aws:s3:::qa-fh-bucket3"


def test_firehose_s3_destination_writes(s3, fh):
    """PutRecord with S3 destination actually writes data to the S3 bucket."""
    import time as _time
    bucket = "fh-s3-dest-v39"
    s3.create_bucket(Bucket=bucket)
    fh.create_delivery_stream(
        DeliveryStreamName="fh-s3-test-v39",
        DeliveryStreamType="DirectPut",
        ExtendedS3DestinationConfiguration={
            "BucketARN": f"arn:aws:s3:::{bucket}",
            "RoleARN": "arn:aws:iam::000000000000:role/firehose",
            "Prefix": "data/",
        },
    )
    fh.put_record(DeliveryStreamName="fh-s3-test-v39", Record={"Data": b"hello from firehose"})
    _time.sleep(1)  # allow async delivery
    objs = s3.list_objects_v2(Bucket=bucket, Prefix="data/")
    assert objs.get("KeyCount", 0) > 0, "Firehose should have written to S3"
    key = objs["Contents"][0]["Key"]
    obj = s3.get_object(Bucket=bucket, Key=key)
    body = obj["Body"].read()
    assert b"hello from firehose" in body


def test_firehose_describe_nonexistent_carries_errortype(fh):
    """Real AWS sends `x-amzn-errortype` on JSON-protocol errors. Java/Go SDK
    v2 read it; without it they raise SdkClientException(unknown error type)."""
    with pytest.raises(ClientError) as exc:
        fh.describe_delivery_stream(DeliveryStreamName="missing-fh")
    assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
    assert exc.value.response["ResponseMetadata"]["HTTPHeaders"].get("x-amzn-errortype") == "ResourceNotFoundException"


def test_firehose_kinesis_stream_as_source_fans_out_to_s3(fh, kin, s3):
    """KinesisStreamAsSource: records put into the source Kinesis stream
    must reach the configured S3 destination. Issue #744."""
    import time as _time
    import uuid as _uuid

    stream_name = f"src-stream-{_uuid.uuid4().hex[:8]}"
    delivery_name = f"fh-{_uuid.uuid4().hex[:8]}"
    bucket = f"fh-src-bucket-{_uuid.uuid4().hex[:8]}"
    s3.create_bucket(Bucket=bucket)
    try:
        kin.create_stream(StreamName=stream_name, ShardCount=1)
        stream_arn = kin.describe_stream(
            StreamName=stream_name)["StreamDescription"]["StreamARN"]
        fh.create_delivery_stream(
            DeliveryStreamName=delivery_name,
            DeliveryStreamType="KinesisStreamAsSource",
            KinesisStreamSourceConfiguration={
                "KinesisStreamARN": stream_arn,
                "RoleARN": "arn:aws:iam::000000000000:role/test",
            },
            ExtendedS3DestinationConfiguration={
                "RoleARN": "arn:aws:iam::000000000000:role/test",
                "BucketARN": f"arn:aws:s3:::{bucket}",
                "Prefix": "out/",
                "CompressionFormat": "UNCOMPRESSED",
            },
        )
        # Single PutRecord.
        kin.put_record(
            StreamName=stream_name, PartitionKey="pk", Data=b'{"a":1}')
        # Batch PutRecords.
        kin.put_records(StreamName=stream_name, Records=[
            {"PartitionKey": "pk1", "Data": b'{"a":2}'},
            {"PartitionKey": "pk2", "Data": b'{"a":3}'},
        ])
        for _ in range(20):
            objs = s3.list_objects_v2(Bucket=bucket, Prefix="out/").get("Contents", [])
            if len(objs) >= 3:
                break
            _time.sleep(0.1)
        objs = s3.list_objects_v2(Bucket=bucket, Prefix="out/").get("Contents", [])
        assert len(objs) >= 3, f"expected 3 records delivered to S3, got {len(objs)}"
        bodies = sorted(
            s3.get_object(Bucket=bucket, Key=o["Key"])["Body"].read()
            for o in objs
        )
        assert bodies == [b'{"a":1}', b'{"a":2}', b'{"a":3}']
    finally:
        try:
            fh.delete_delivery_stream(DeliveryStreamName=delivery_name)
        except Exception:
            pass
        try:
            kin.delete_stream(StreamName=stream_name)
        except Exception:
            pass


# ---------------------------------------------------------------------------
# ProcessingConfiguration — Lambda processor invocation.
#
# Real AWS Firehose invokes the configured Lambda for every batch of records
# in flight, with payload shape:
#   {invocationId, deliveryStreamArn, region, records:[{recordId, approximateArrivalTimestamp, data(base64)}]}
# and expects a response shape:
#   {records:[{recordId, result:"Ok"|"Dropped"|"ProcessingFailed", data(base64)}]}
# Records marked Dropped/ProcessingFailed are not written to the destination;
# Ok records are written with the transformed data. On any Lambda failure
# Firehose passes records through unchanged (best-effort by AWS contract).
# ---------------------------------------------------------------------------


def _processor_lambda_zip(handler_src: str) -> bytes:
    buf = io.BytesIO()
    with zipfile.ZipFile(buf, "w") as zf:
        zf.writestr("index.py", handler_src)
    return buf.getvalue()


def _make_processor_fn(lam, name: str, handler_src: str) -> str:
    lam.create_function(
        FunctionName=name,
        Runtime="python3.12",
        Role="arn:aws:iam::000000000000:role/r",
        Handler="index.handler",
        Code={"ZipFile": _processor_lambda_zip(handler_src)},
    )
    return f"arn:aws:lambda:us-east-1:000000000000:function:{name}"


def _create_firehose_with_processor(fh, name: str, bucket: str, lambda_arn: str,
                                    delivery_type: str = "DirectPut",
                                    kinesis_arn: str | None = None) -> None:
    ext_s3 = {
        "BucketARN": f"arn:aws:s3:::{bucket}",
        "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
        "BufferingHints": {"SizeInMBs": 1, "IntervalInSeconds": 1},
        "CompressionFormat": "UNCOMPRESSED",
        "Prefix": "out/",
        "ProcessingConfiguration": {
            "Enabled": True,
            "Processors": [{
                "Type": "Lambda",
                "Parameters": [
                    {"ParameterName": "LambdaArn", "ParameterValue": lambda_arn},
                ],
            }],
        },
    }
    if delivery_type == "KinesisStreamAsSource" and kinesis_arn:
        fh.create_delivery_stream(
            DeliveryStreamName=name,
            DeliveryStreamType="KinesisStreamAsSource",
            KinesisStreamSourceConfiguration={
                "KinesisStreamARN": kinesis_arn,
                "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
            },
            ExtendedS3DestinationConfiguration=ext_s3,
        )
    else:
        fh.create_delivery_stream(
            DeliveryStreamName=name,
            DeliveryStreamType="DirectPut",
            ExtendedS3DestinationConfiguration=ext_s3,
        )


def test_firehose_lambda_processor_transforms_record(fh, s3, lam):
    """Lambda processor returns transformed data → S3 object contains the
    transform output, not the original record."""
    suffix = _uuid_mod.uuid4().hex[:8]
    bucket = f"fh-proc-{suffix}"
    fn = f"fh-proc-fn-{suffix}"
    delivery = f"fh-proc-{suffix}"
    s3.create_bucket(Bucket=bucket)
    handler = (
        "import base64, json\n"
        "def handler(event, context):\n"
        "    out = []\n"
        "    for r in event['records']:\n"
        "        raw = base64.b64decode(r['data'])\n"
        "        transformed = raw.upper()\n"
        "        out.append({\n"
        "            'recordId': r['recordId'],\n"
        "            'result': 'Ok',\n"
        "            'data': base64.b64encode(transformed).decode(),\n"
        "        })\n"
        "    return {'records': out}\n"
    )
    arn = _make_processor_fn(lam, fn, handler)
    _create_firehose_with_processor(fh, delivery, bucket, arn)
    try:
        fh.put_record(DeliveryStreamName=delivery, Record={"Data": b"hello"})
        for _ in range(20):
            objs = s3.list_objects_v2(Bucket=bucket, Prefix="out/").get("Contents", [])
            if objs:
                break
            time.sleep(0.1)
        objs = s3.list_objects_v2(Bucket=bucket, Prefix="out/").get("Contents", [])
        assert len(objs) == 1, f"expected 1 transformed record, got {len(objs)}"
        body = s3.get_object(Bucket=bucket, Key=objs[0]["Key"])["Body"].read()
        assert body == b"HELLO", body
    finally:
        try:
            fh.delete_delivery_stream(DeliveryStreamName=delivery)
        except Exception:
            pass
        try:
            lam.delete_function(FunctionName=fn)
        except Exception:
            pass


def test_firehose_lambda_processor_dropped_record_not_written(fh, s3, lam):
    """``result: Dropped`` → record is filtered out, no S3 object written."""
    suffix = _uuid_mod.uuid4().hex[:8]
    bucket = f"fh-drop-{suffix}"
    fn = f"fh-drop-fn-{suffix}"
    delivery = f"fh-drop-{suffix}"
    s3.create_bucket(Bucket=bucket)
    handler = (
        "def handler(event, context):\n"
        "    return {'records': [\n"
        "        {'recordId': r['recordId'], 'result': 'Dropped', 'data': r['data']}\n"
        "        for r in event['records']\n"
        "    ]}\n"
    )
    arn = _make_processor_fn(lam, fn, handler)
    _create_firehose_with_processor(fh, delivery, bucket, arn)
    try:
        fh.put_record(DeliveryStreamName=delivery, Record={"Data": b"hello"})
        time.sleep(0.6)  # give the fire-and-forget S3 task time
        objs = s3.list_objects_v2(Bucket=bucket, Prefix="out/").get("Contents", [])
        assert objs == [], f"expected no S3 objects, got {objs}"
    finally:
        try:
            fh.delete_delivery_stream(DeliveryStreamName=delivery)
        except Exception:
            pass
        try:
            lam.delete_function(FunctionName=fn)
        except Exception:
            pass


def test_firehose_lambda_processor_not_found_passes_through(fh, s3):
    """Configured LambdaArn that doesn't exist → record delivered unchanged
    (Firehose is best-effort; processor failure must not break the stream)."""
    suffix = _uuid_mod.uuid4().hex[:8]
    bucket = f"fh-miss-{suffix}"
    delivery = f"fh-miss-{suffix}"
    s3.create_bucket(Bucket=bucket)
    missing_arn = (
        f"arn:aws:lambda:us-east-1:000000000000:function:fh-no-such-{suffix}"
    )
    _create_firehose_with_processor(fh, delivery, bucket, missing_arn)
    try:
        fh.put_record(DeliveryStreamName=delivery, Record={"Data": b"untouched"})
        for _ in range(20):
            objs = s3.list_objects_v2(Bucket=bucket, Prefix="out/").get("Contents", [])
            if objs:
                break
            time.sleep(0.1)
        objs = s3.list_objects_v2(Bucket=bucket, Prefix="out/").get("Contents", [])
        assert len(objs) == 1
        body = s3.get_object(Bucket=bucket, Key=objs[0]["Key"])["Body"].read()
        assert body == b"untouched"
    finally:
        try:
            fh.delete_delivery_stream(DeliveryStreamName=delivery)
        except Exception:
            pass


def test_firehose_kinesis_source_with_lambda_processor(fh, s3, lam, kin):
    """End-to-end Kinesis → Firehose → Lambda processor → S3 — the user's
    exact repro from issue #744 follow-up."""
    suffix = _uuid_mod.uuid4().hex[:8]
    bucket = f"fh-kin-proc-{suffix}"
    fn = f"fh-kin-proc-fn-{suffix}"
    delivery = f"fh-kin-proc-{suffix}"
    stream = f"fh-kin-proc-src-{suffix}"
    s3.create_bucket(Bucket=bucket)
    kin.create_stream(StreamName=stream, ShardCount=1)
    # Wait for stream to be ACTIVE.
    for _ in range(20):
        if kin.describe_stream(StreamName=stream)["StreamDescription"]["StreamStatus"] == "ACTIVE":
            break
        time.sleep(0.1)
    stream_arn = kin.describe_stream(StreamName=stream)["StreamDescription"]["StreamARN"]
    handler = (
        "import base64\n"
        "def handler(event, context):\n"
        "    out = []\n"
        "    for r in event['records']:\n"
        "        raw = base64.b64decode(r['data'])\n"
        "        out.append({\n"
        "            'recordId': r['recordId'],\n"
        "            'result': 'Ok',\n"
        "            'data': base64.b64encode(b'proc:' + raw).decode(),\n"
        "        })\n"
        "    return {'records': out}\n"
    )
    arn = _make_processor_fn(lam, fn, handler)
    _create_firehose_with_processor(
        fh, delivery, bucket, arn,
        delivery_type="KinesisStreamAsSource", kinesis_arn=stream_arn,
    )
    try:
        kin.put_record(StreamName=stream, PartitionKey="pk", Data=b"payload")
        for _ in range(30):
            objs = s3.list_objects_v2(Bucket=bucket, Prefix="out/").get("Contents", [])
            if objs:
                break
            time.sleep(0.1)
        objs = s3.list_objects_v2(Bucket=bucket, Prefix="out/").get("Contents", [])
        assert len(objs) >= 1, "expected at least 1 record delivered to S3"
        body = s3.get_object(Bucket=bucket, Key=objs[0]["Key"])["Body"].read()
        assert body == b"proc:payload", body
    finally:
        try:
            fh.delete_delivery_stream(DeliveryStreamName=delivery)
        except Exception:
            pass
        try:
            lam.delete_function(FunctionName=fn)
        except Exception:
            pass
        try:
            kin.delete_stream(StreamName=stream)
        except Exception:
            pass


def test_firehose_same_name_streams_are_region_scoped(fh):
    """Same-named delivery streams in two regions stay isolated with region-bearing ARNs."""
    west = _regional_client("firehose", "us-west-2")
    name = f"intg-fh-region-scope-{_uuid_mod.uuid4().hex[:8]}"

    east_arn = fh.create_delivery_stream(
        DeliveryStreamName=name, DeliveryStreamType="DirectPut"
    )["DeliveryStreamARN"]
    west_arn = west.create_delivery_stream(
        DeliveryStreamName=name, DeliveryStreamType="DirectPut"
    )["DeliveryStreamARN"]

    assert east_arn == f"arn:aws:firehose:us-east-1:000000000000:deliverystream/{name}"
    assert west_arn == f"arn:aws:firehose:us-west-2:000000000000:deliverystream/{name}"

    east_desc = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    west_desc = west.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
    assert east_desc["DeliveryStreamARN"] == east_arn
    assert west_desc["DeliveryStreamARN"] == west_arn

    # Use a high Limit so the assertion is robust to other streams the shared
    # session fixture accumulates (default page size is 10).
    assert name in fh.list_delivery_streams(Limit=10000)["DeliveryStreamNames"]
    assert name in west.list_delivery_streams(Limit=10000)["DeliveryStreamNames"]


def test_firehose_stream_names_do_not_fallback_across_region(fh):
    """A stream created in one region is not visible from another region."""
    west = _regional_client("firehose", "us-west-2")
    name = f"intg-fh-no-fallback-{_uuid_mod.uuid4().hex[:8]}"

    west.create_delivery_stream(DeliveryStreamName=name, DeliveryStreamType="DirectPut")

    with pytest.raises(ClientError) as exc:
        fh.describe_delivery_stream(DeliveryStreamName=name)
    assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"

    # West still resolves it, and it never leaks into the east listing.
    assert name in west.list_delivery_streams(Limit=10000)["DeliveryStreamNames"]
    assert name not in fh.list_delivery_streams(Limit=10000)["DeliveryStreamNames"]


def test_firehose_restore_legacy_account_scoped_state_uses_arn_region():
    """Legacy account-scoped persisted streams migrate to the region in their ARN."""
    from ministack.core.responses import (
        AccountScopedDict,
        get_account_id,
        get_region,
        set_request_account_id,
        set_request_region,
    )
    from ministack.services import firehose as _fh

    original_account = get_account_id()
    original_region = get_region()
    account_id = "000000000000"
    name = "LegacyFirehoseStream"
    stream_arn = f"arn:aws:firehose:us-west-2:{account_id}:deliverystream/{name}"

    legacy_streams = AccountScopedDict()
    legacy_streams._data[(account_id, name)] = {
        "name": name,
        "arn": stream_arn,
        "status": "ACTIVE",
        "type": "DirectPut",
        "version": 1,
        "created_at": 1700000000,
        "updated_at": 1700000000,
        "destinations": [],
        "tags": {},
        "encryption": None,
        "kinesis_source": None,
    }

    _fh.reset()
    try:
        set_request_account_id(account_id)
        set_request_region("us-east-1")

        _fh.load_persisted_state({"_streams": legacy_streams})

        # The legacy stream lands in the region carried by its ARN, not the
        # active request region.
        assert _fh._streams.get_scoped(account_id, "us-east-1", name) is None
        assert _fh._streams.get_scoped(account_id, "us-west-2", name)["arn"] == stream_arn
    finally:
        _fh.reset()
        set_request_account_id(original_account)
        set_request_region(original_region)


def test_firehose_kinesis_source_ingest_is_region_scoped():
    """ingest_from_kinesis_source only fans out to same-region delivery streams.

    A KinesisStreamAsSource delivery stream registered in us-west-2 must NOT be
    matched by a Kinesis-source ingest running under a different request region,
    even when the source ARN matches. Exercises the regional filtering inside
    ``ingest_from_kinesis_source`` (``_streams.values()`` is scoped to the
    current account+region), which the positive same-region tests don't cover.
    """
    from ministack.core.responses import (
        get_account_id,
        get_region,
        set_request_account_id,
        set_request_region,
    )
    from ministack.services import firehose as _fh

    original_account = get_account_id()
    original_region = get_region()
    account_id = "000000000000"
    name = "KinSrcRegionScoped"
    source_arn = f"arn:aws:kinesis:us-west-2:{account_id}:stream/src"

    def _make_stream():
        return {
            "name": name,
            "arn": f"arn:aws:firehose:us-west-2:{account_id}:deliverystream/{name}",
            "status": "ACTIVE",
            "type": "KinesisStreamAsSource",
            "version": 1,
            "created_at": 1700000000,
            "updated_at": 1700000000,
            "destinations": [
                {
                    "id": "destinationId-000000000001",
                    "type": "ExtendedS3",
                    "config": {"BucketARN": "arn:aws:s3:::no-such-bucket", "Prefix": "out/"},
                    "records": [],
                }
            ],
            "tags": {},
            "encryption": None,
            "kinesis_source": {
                "KinesisStreamARN": source_arn,
                "RoleARN": "arn:aws:iam::000000000000:role/test",
                "DeliveryStartTimestamp": 0,
            },
        }

    _fh.reset()
    try:
        set_request_account_id(account_id)

        # Register the delivery stream in us-west-2.
        set_request_region("us-west-2")
        west_stream = _make_stream()
        _fh._streams.set_scoped(account_id, "us-west-2", name, west_stream)

        # Ingest under a DIFFERENT region: the us-west-2 stream is out of scope,
        # so nothing is delivered even though the source ARN matches.
        set_request_region("us-east-1")
        _fh.ingest_from_kinesis_source(source_arn, [("pk", b'{"a":1}')])
        assert west_stream["destinations"][0]["records"] == []

        # Control: same-region ingest matches and appends the record.
        set_request_region("us-west-2")
        _fh.ingest_from_kinesis_source(source_arn, [("pk", b'{"a":1}')])
        assert len(west_stream["destinations"][0]["records"]) == 1
    finally:
        _fh.reset()
        set_request_account_id(original_account)
        set_request_region(original_region)


def _iceberg_test_con(bucket_arn):
    """DuckDB connection attached to MiniStack's S3 Tables Iceberg REST catalog.
    Skips the test if the iceberg/httpfs extensions or REST-catalog writes are
    unavailable in this environment (e.g. offline CI or an old DuckDB)."""
    import duckdb

    parsed = urlparse(ENDPOINT)
    host = parsed.hostname or "localhost"
    port = parsed.port or 4566
    con = duckdb.connect()
    try:
        con.execute("INSTALL iceberg; LOAD iceberg; INSTALL httpfs; LOAD httpfs;")
        con.execute(
            f"CREATE SECRET s (TYPE S3, KEY_ID 'test', SECRET 'test', "
            f"ENDPOINT '{host}:{port}', URL_STYLE 'path', USE_SSL false, "
            f"REGION 'us-east-1')"
        )
        con.execute(
            f"ATTACH '{bucket_arn}' AS cat (TYPE ICEBERG, "
            f"ENDPOINT '{ENDPOINT}/iceberg', AUTHORIZATION_TYPE 'none')"
        )
    except Exception as exc:  # pragma: no cover - environment dependent
        con.close()
        pytest.skip(f"DuckDB Iceberg REST writes unavailable: {exc}")
    return con


def test_firehose_iceberg_delivery_writes_queryable_rows(fh):
    """DirectPut records delivered to an Iceberg (S3 Tables) destination are
    written into the table via the Iceberg REST catalog and are queryable,
    matching real Amazon Data Firehose. Operation defaults to insert."""
    suffix = _uuid_mod.uuid4().hex[:8]
    s3t = _regional_client("s3tables", "us-east-1")
    bucket = s3t.create_table_bucket(name=f"fh-ice-{suffix}")["arn"]
    s3t.create_namespace(tableBucketARN=bucket, namespace=["analytics"])
    con = _iceberg_test_con(bucket)
    con.execute("CREATE TABLE cat.analytics.events (id INTEGER, name VARCHAR)")
    con.close()

    stream = f"fh-ice-{suffix}"
    fh.create_delivery_stream(
        DeliveryStreamName=stream,
        DeliveryStreamType="DirectPut",
        IcebergDestinationConfiguration={
            "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
            "CatalogConfiguration": {"CatalogARN": bucket},
            "S3Configuration": {
                "BucketARN": "arn:aws:s3:::fh-ice-errors",
                "RoleARN": "arn:aws:iam::000000000000:role/firehose-role",
            },
            "DestinationTableConfigurationList": [{
                "DestinationDatabaseName": "analytics",
                "DestinationTableName": "events",
                "UniqueKeys": ["id"],
            }],
        },
    )
    try:
        fh.put_record(
            DeliveryStreamName=stream,
            Record={"Data": json.dumps({"id": 1, "name": "alice"}).encode()},
        )
        rows = None
        for _ in range(60):
            try:
                q = _iceberg_test_con(bucket)
                rows = q.execute(
                    "SELECT id, name FROM cat.analytics.events ORDER BY id"
                ).fetchall()
                q.close()
                if rows == [(1, "alice")]:
                    break
            except Exception:
                pass
            time.sleep(0.5)
        assert rows == [(1, "alice")]
    finally:
        fh.delete_delivery_stream(DeliveryStreamName=stream)


def _wait_objects(s3, bucket, prefix, count=1):
    for _ in range(40):
        objs = s3.list_objects_v2(Bucket=bucket, Prefix=prefix).get("Contents", [])
        if len(objs) >= count:
            return objs
        time.sleep(0.1)
    return s3.list_objects_v2(Bucket=bucket, Prefix=prefix).get("Contents", [])


def _partitioned_parquet_stream(fh, glue, s3, suffix, query):
    bucket, database, name = f"fh-dp-{suffix}", f"fh_dp_{suffix}", f"fh-dp-{suffix}"
    s3.create_bucket(Bucket=bucket)
    glue.create_database(DatabaseInput={"Name": database})
    glue.create_table(DatabaseName=database, TableInput={"Name": "events", "StorageDescriptor": {
        "Columns": [{"Name": "timestamp", "Type": "string"}, {"Name": "action", "Type": "int"}],
        "Location": f"s3://{bucket}/usage/"}})
    fh.create_delivery_stream(DeliveryStreamName=name, DeliveryStreamType="DirectPut",
                              ExtendedS3DestinationConfiguration={
        "RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": f"arn:aws:s3:::{bucket}",
        "Prefix": "usage/year=!{partitionKeyFromQuery:year}/month=!{partitionKeyFromQuery:month}/"
                  "day=!{partitionKeyFromQuery:day}/",
        "ErrorOutputPrefix": "errors/!{firehose:error-output-type}/",
        "DynamicPartitioningConfiguration": {"Enabled": True},
        "ProcessingConfiguration": {"Enabled": True, "Processors": [{"Type": "MetadataExtraction", "Parameters": [
            {"ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6"},
            {"ParameterName": "MetadataExtractionQuery", "ParameterValue": query}]}]},
        "DataFormatConversionConfiguration": {"Enabled": True, "SchemaConfiguration": {
            "RoleARN": "arn:aws:iam::000000000000:role/firehose", "DatabaseName": database,
            "TableName": "events", "Region": "us-east-1"},
            "InputFormatConfiguration": {"Deserializer": {"HiveJsonSerDe": {}}},
            "OutputFormatConfiguration": {"Serializer": {"ParquetSerDe": {"Compression": "SNAPPY"}}}},
    })
    return bucket, name


def test_firehose_dynamic_partitioning_writes_parquet(fh, glue, s3, tmp_path):
    duckdb = pytest.importorskip("duckdb")
    bucket, name = _partitioned_parquet_stream(
        fh, glue, s3, _uuid_mod.uuid4().hex[:8],
        "{year: .timestamp[0:4], month: .timestamp[5:7], day: .timestamp[8:10]}")
    fh.put_record(DeliveryStreamName=name,
                  Record={"Data": json.dumps({"timestamp": "2026-04-23T11:30:00Z", "action": 1})})
    objs = _wait_objects(s3, bucket, "usage/")
    assert len(objs) == 1, objs
    key = objs[0]["Key"]
    # No timestamp expression in the prefix, so Firehose appends yyyy/MM/dd/HH/.
    assert re.fullmatch(rf"usage/year=2026/month=04/day=23/\d{{4}}/\d{{2}}/\d{{2}}/\d{{2}}/{name}-1-"
                        r"\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}-[0-9a-f-]{36}\.parquet", key), key
    body = s3.get_object(Bucket=bucket, Key=key)["Body"].read()
    assert body[:4] == b"PAR1" and body[-4:] == b"PAR1"
    path = tmp_path / "out.parquet"
    path.write_bytes(body)
    con = duckdb.connect()
    assert con.execute(f"SELECT * FROM '{path}'").fetchall() == [("2026-04-23T11:30:00Z", 1)]
    assert con.execute(f"SELECT DISTINCT compression FROM parquet_metadata('{path}')").fetchall() == [("SNAPPY",)]


def test_firehose_failed_records_go_to_the_error_prefix(fh, glue, s3):
    pytest.importorskip("duckdb")
    bucket, name = _partitioned_parquet_stream(
        fh, glue, s3, _uuid_mod.uuid4().hex[:8], "{year: .timestamp[0:4], month: .month, day: .day}")
    missing_key = json.dumps({"timestamp": "2026-04-23T11:30:00Z", "action": 1})
    fh.put_record(DeliveryStreamName=name, Record={"Data": missing_key})
    failed = _wait_objects(s3, bucket, "errors/processing-failed/")
    assert len(failed) == 1, failed
    document = json.loads(s3.get_object(Bucket=bucket, Key=failed[0]["Key"])["Body"].read())
    assert base64.b64decode(document["rawData"]).decode() == missing_key
    assert {"attemptsMade", "arrivalTimestamp", "attemptEndingTimestamp", "ErrorMessage"} <= set(document)

    bad_type = json.dumps({"timestamp": "2026-04-23T11:30:00Z", "action": "abc", "month": "04", "day": "23"})
    fh.put_record(DeliveryStreamName=name, Record={"Data": bad_type})
    failed = _wait_objects(s3, bucket, "errors/format-conversion-failed/")
    assert len(failed) == 1, failed
    document = json.loads(s3.get_object(Bucket=bucket, Key=failed[0]["Key"])["Body"].read())
    assert document["ErrorCode"] == "DataFormatConversion.MalformedData"
    assert document["ErrorMessage"] == "Data does not match the schema."
    assert document["dataCatalogTable"]["tableName"] == "events"
    assert not s3.list_objects_v2(Bucket=bucket, Prefix="usage/").get("Contents")


def test_firehose_prefix_timestamp_expressions_replace_the_default(fh, s3):
    suffix = _uuid_mod.uuid4().hex[:8]
    bucket, name = f"fh-ts-{suffix}", f"fh-ts-{suffix}"
    s3.create_bucket(Bucket=bucket)
    fh.create_delivery_stream(DeliveryStreamName=name, ExtendedS3DestinationConfiguration={
        "RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": f"arn:aws:s3:::{bucket}",
        "Prefix": "logs/year=!{timestamp:yyyy}/", "ErrorOutputPrefix": "errors/!{firehose:error-output-type}/"})
    fh.put_record(DeliveryStreamName=name, Record={"Data": b"raw"})
    objs = _wait_objects(s3, bucket, "logs/")
    assert len(objs) == 1
    assert re.fullmatch(rf"logs/year=\d{{4}}/{name}-1-\d{{4}}(-\d{{2}}){{5}}-[0-9a-f-]{{36}}", objs[0]["Key"]), objs
    assert s3.get_object(Bucket=bucket, Key=objs[0]["Key"])["Body"].read() == b"raw"


def test_firehose_partition_keys_from_lambda(fh, s3, lam):
    suffix = _uuid_mod.uuid4().hex[:8]
    bucket, fn, name = f"fh-lk-{suffix}", f"fh-lk-fn-{suffix}", f"fh-lk-{suffix}"
    s3.create_bucket(Bucket=bucket)
    handler = (
        "import base64, json\n"
        "def handler(event, context):\n"
        "    out = []\n"
        "    for r in event['records']:\n"
        "        doc = json.loads(base64.b64decode(r['data']))\n"
        "        out.append({'recordId': r['recordId'], 'result': 'Ok', 'data': r['data'],\n"
        "                    'metadata': {'partitionKeys': {'customer': doc['customer']}}})\n"
        "    return {'records': out}\n"
    )
    arn = _make_processor_fn(lam, fn, handler)
    fh.create_delivery_stream(DeliveryStreamName=name, ExtendedS3DestinationConfiguration={
        "RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": f"arn:aws:s3:::{bucket}",
        "Prefix": "c=!{partitionKeyFromLambda:customer}/", "ErrorOutputPrefix": "errors/!{firehose:error-output-type}/",
        "DynamicPartitioningConfiguration": {"Enabled": True},
        "ProcessingConfiguration": {"Enabled": True, "Processors": [{"Type": "Lambda", "Parameters": [
            {"ParameterName": "LambdaArn", "ParameterValue": arn}]}]}})
    try:
        fh.put_record(DeliveryStreamName=name, Record={"Data": json.dumps({"customer": "acme"})})
        objs = _wait_objects(s3, bucket, "c=acme/")
        assert len(objs) == 1, s3.list_objects_v2(Bucket=bucket).get("Contents")
    finally:
        fh.delete_delivery_stream(DeliveryStreamName=name)
        lam.delete_function(FunctionName=fn)


def test_firehose_region_delivery_into_regional_bucket():
    """A stream in eu-west-1 delivers into an eu-west-1 bucket; the internal
    S3 write is unsigned, so the bucket home-region check doesn't apply."""
    suffix = _uuid_mod.uuid4().hex[:8]
    bucket, name = f"fh-euw1-{suffix}", f"fh-euw1-{suffix}"
    fh, s3 = _regional_client("firehose", "eu-west-1"), _regional_client("s3", "eu-west-1")
    s3.create_bucket(Bucket=bucket, CreateBucketConfiguration={"LocationConstraint": "eu-west-1"})
    fh.create_delivery_stream(DeliveryStreamName=name, DeliveryStreamType="DirectPut",
                              ExtendedS3DestinationConfiguration={
                                  "RoleARN": "arn:aws:iam::000000000000:role/firehose",
                                  "BucketARN": f"arn:aws:s3:::{bucket}", "Prefix": "out/",
                                  "BufferingHints": {"SizeInMBs": 1, "IntervalInSeconds": 1}})
    try:
        fh.put_record(DeliveryStreamName=name, Record={"Data": b"regional"})
        objs = _wait_objects(s3, bucket, "out/")
        assert len(objs) == 1, objs
        assert s3.get_object(Bucket=bucket, Key=objs[0]["Key"])["Body"].read() == b"regional"
    finally:
        fh.delete_delivery_stream(DeliveryStreamName=name)
        for obj in s3.list_objects_v2(Bucket=bucket).get("Contents", []):
            s3.delete_object(Bucket=bucket, Key=obj["Key"])
        s3.delete_bucket(Bucket=bucket)