Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
import asyncio
import base64
import io
import json
import os
import time
import uuid as _uuid_mod
import zipfile
from types import SimpleNamespace
import boto3
import cbor2
import pytest
from botocore.config import Config
from botocore.eventstream import EventStreamBuffer
from botocore.exceptions import ClientError
from ministack.core.responses import AccountRegionScopedDict, request_scope
from ministack.services import kinesis
_LAMBDA_ROLE = "arn:aws:iam::000000000000:role/lambda-role"
def _regional_kin(region_name, account_id="test"):
endpoint = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566")
return boto3.client(
"kinesis",
endpoint_url=endpoint,
aws_access_key_id=account_id,
aws_secret_access_key="test",
region_name=region_name,
config=Config(region_name=region_name, retries={"mode": "standard"}),
)
def test_kinesis_put_get(kin):
kin.create_stream(StreamName="test-stream", ShardCount=1)
kin.put_record(StreamName="test-stream", Data=b"hello kinesis", PartitionKey="pk1")
kin.put_record(StreamName="test-stream", Data=b"second record", PartitionKey="pk2")
desc = kin.describe_stream(StreamName="test-stream")
shard_id = desc["StreamDescription"]["Shards"][0]["ShardId"]
it = kin.get_shard_iterator(StreamName="test-stream", ShardId=shard_id, ShardIteratorType="TRIM_HORIZON")
records = kin.get_records(ShardIterator=it["ShardIterator"])
assert len(records["Records"]) == 2
def test_kinesis_batch(kin):
kin.create_stream(StreamName="test-stream-batch", ShardCount=1)
resp = kin.put_records(
StreamName="test-stream-batch",
Records=[{"Data": f"record-{i}".encode(), "PartitionKey": f"pk{i}"} for i in range(5)],
)
assert resp["FailedRecordCount"] == 0
assert len(resp["Records"]) == 5
def test_kinesis_list(kin):
resp = kin.list_streams()
assert "test-stream" in resp["StreamNames"]
def test_kinesis_create_stream_v2(kin):
kin.create_stream(StreamName="kin-cs-v2", ShardCount=2)
desc = kin.describe_stream(StreamName="kin-cs-v2")
sd = desc["StreamDescription"]
assert sd["StreamName"] == "kin-cs-v2"
assert sd["StreamStatus"] == "ACTIVE"
assert len(sd["Shards"]) == 2
def test_kinesis_put_get_records_v2(kin):
kin.create_stream(StreamName="kin-pgr-v2", ShardCount=1)
kin.put_record(StreamName="kin-pgr-v2", Data=b"rec1", PartitionKey="pk1")
kin.put_record(StreamName="kin-pgr-v2", Data=b"rec2", PartitionKey="pk2")
kin.put_record(StreamName="kin-pgr-v2", Data=b"rec3", PartitionKey="pk3")
desc = kin.describe_stream(StreamName="kin-pgr-v2")
shard_id = desc["StreamDescription"]["Shards"][0]["ShardId"]
it = kin.get_shard_iterator(
StreamName="kin-pgr-v2",
ShardId=shard_id,
ShardIteratorType="TRIM_HORIZON",
)
records = kin.get_records(ShardIterator=it["ShardIterator"])
assert len(records["Records"]) == 3
assert records["Records"][0]["Data"] == b"rec1"
def test_kinesis_put_records_batch_v2(kin):
kin.create_stream(StreamName="kin-batch-v2", ShardCount=1)
resp = kin.put_records(
StreamName="kin-batch-v2",
Records=[{"Data": f"b{i}".encode(), "PartitionKey": f"pk{i}"} for i in range(7)],
)
assert resp["FailedRecordCount"] == 0
assert len(resp["Records"]) == 7
for r in resp["Records"]:
assert "ShardId" in r
assert "SequenceNumber" in r
def test_kinesis_list_streams_v2(kin):
kin.create_stream(StreamName="kin-ls-v2a", ShardCount=1)
kin.create_stream(StreamName="kin-ls-v2b", ShardCount=1)
resp = kin.list_streams()
assert "kin-ls-v2a" in resp["StreamNames"]
assert "kin-ls-v2b" in resp["StreamNames"]
def test_kinesis_list_shards_v2(kin):
kin.create_stream(StreamName="kin-lsh-v2", ShardCount=3)
resp = kin.list_shards(StreamName="kin-lsh-v2")
assert len(resp["Shards"]) == 3
for shard in resp["Shards"]:
assert "ShardId" in shard
assert "HashKeyRange" in shard
def test_kinesis_describe_stream_v2(kin):
kin.create_stream(StreamName="kin-desc-v2", ShardCount=1)
resp = kin.describe_stream(StreamName="kin-desc-v2")
sd = resp["StreamDescription"]
assert sd["StreamName"] == "kin-desc-v2"
assert sd["RetentionPeriodHours"] == 24
assert "StreamARN" in sd
assert len(sd["Shards"]) == 1
summary = kin.describe_stream_summary(StreamName="kin-desc-v2")
assert summary["StreamDescriptionSummary"]["StreamName"] == "kin-desc-v2"
def test_kinesis_stream_arn_rejects_foreign_region_without_name_fallback(kin):
"""StreamARN inputs must be parsed before matching account-scoped state."""
west = _regional_kin("us-west-2")
stream_name = f"kin-arn-scope-{_uuid_mod.uuid4().hex[:8]}"
west.create_stream(StreamName=stream_name, ShardCount=1)
west_arn = west.describe_stream(StreamName=stream_name)["StreamDescription"]["StreamARN"]
with pytest.raises(ClientError) as exc:
kin.describe_stream(StreamARN=west_arn)
assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
with pytest.raises(ClientError) as exc:
kin.put_record(StreamARN=west_arn, Data=b"no-fallback", PartitionKey="pk")
assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
def test_kinesis_same_name_streams_are_region_scoped(kin):
west = _regional_kin("us-west-2")
stream_name = f"kin-region-scope-{_uuid_mod.uuid4().hex[:8]}"
kin.create_stream(StreamName=stream_name, ShardCount=1)
west.create_stream(StreamName=stream_name, ShardCount=1)
east_desc = kin.describe_stream(StreamName=stream_name)["StreamDescription"]
west_desc = west.describe_stream(StreamName=stream_name)["StreamDescription"]
assert east_desc["StreamARN"] == f"arn:aws:kinesis:us-east-1:000000000000:stream/{stream_name}"
assert west_desc["StreamARN"] == f"arn:aws:kinesis:us-west-2:000000000000:stream/{stream_name}"
assert stream_name in kin.list_streams()["StreamNames"]
assert stream_name in west.list_streams()["StreamNames"]
def test_kinesis_stream_names_do_not_fallback_across_region_or_account(kin):
owner_account = "111111111111"
west_owner = _regional_kin("us-west-2", account_id=owner_account)
east_owner = _regional_kin("us-east-1", account_id=owner_account)
west_default = _regional_kin("us-west-2")
stream_name = f"kin-no-fallback-{_uuid_mod.uuid4().hex[:8]}"
west_owner.create_stream(StreamName=stream_name, ShardCount=1)
west_arn = west_owner.describe_stream(StreamName=stream_name)["StreamDescription"]["StreamARN"]
with pytest.raises(ClientError) as exc:
east_owner.describe_stream(StreamName=stream_name)
assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
with pytest.raises(ClientError) as exc:
west_default.describe_stream(StreamName=stream_name)
assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
with pytest.raises(ClientError) as exc:
kin.describe_stream(StreamARN=west_arn)
assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
def test_kinesis_tags_v2(kin):
kin.create_stream(StreamName="kin-tag-v2", ShardCount=1)
kin.add_tags_to_stream(StreamName="kin-tag-v2", Tags={"env": "test", "team": "data"})
resp = kin.list_tags_for_stream(StreamName="kin-tag-v2")
tag_map = {t["Key"]: t["Value"] for t in resp["Tags"]}
assert tag_map["env"] == "test"
assert tag_map["team"] == "data"
kin.remove_tags_from_stream(StreamName="kin-tag-v2", TagKeys=["team"])
resp2 = kin.list_tags_for_stream(StreamName="kin-tag-v2")
tag_map2 = {t["Key"]: t["Value"] for t in resp2["Tags"]}
assert "team" not in tag_map2
assert tag_map2["env"] == "test"
def test_kinesis_delete_stream_v2(kin):
kin.create_stream(StreamName="kin-del-v2", ShardCount=1)
kin.delete_stream(StreamName="kin-del-v2")
with pytest.raises(ClientError) as exc:
kin.describe_stream(StreamName="kin-del-v2")
assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
# Real AWS sends `x-amzn-errortype` on JSON-protocol errors; Java/Go SDK v2 read it.
assert exc.value.response["ResponseMetadata"]["HTTPHeaders"].get("x-amzn-errortype") == "ResourceNotFoundException"
def test_kinesis_stream_encryption(kin):
import uuid as _uuid
sname = f"intg-enc-str-{_uuid.uuid4().hex[:8]}"
kin.create_stream(StreamName=sname, ShardCount=1)
time.sleep(0.5)
kin.start_stream_encryption(StreamName=sname, EncryptionType="KMS", KeyId="alias/aws/kinesis")
resp = kin.describe_stream(StreamName=sname)
assert resp["StreamDescription"]["EncryptionType"] == "KMS"
kin.stop_stream_encryption(StreamName=sname, EncryptionType="KMS", KeyId="alias/aws/kinesis")
resp2 = kin.describe_stream(StreamName=sname)
assert resp2["StreamDescription"]["EncryptionType"] == "NONE"
kin.delete_stream(StreamName=sname)
def test_kinesis_enhanced_monitoring(kin):
import uuid as _uuid
sname = f"intg-mon-str-{_uuid.uuid4().hex[:8]}"
kin.create_stream(StreamName=sname, ShardCount=1)
time.sleep(0.5)
resp = kin.enable_enhanced_monitoring(StreamName=sname, ShardLevelMetrics=["IncomingBytes", "OutgoingBytes"])
assert "IncomingBytes" in resp.get("DesiredShardLevelMetrics", [])
resp2 = kin.disable_enhanced_monitoring(StreamName=sname, ShardLevelMetrics=["IncomingBytes"])
assert "IncomingBytes" not in resp2.get("DesiredShardLevelMetrics", [])
kin.delete_stream(StreamName=sname)
def test_kinesis_split_shard(kin):
import uuid as _uuid
sname = f"intg-split-{_uuid.uuid4().hex[:8]}"
kin.create_stream(StreamName=sname, ShardCount=1)
time.sleep(0.3)
desc = kin.describe_stream(StreamName=sname)
shard_id = desc["StreamDescription"]["Shards"][0]["ShardId"]
start_hash = int(desc["StreamDescription"]["Shards"][0]["HashKeyRange"]["StartingHashKey"])
end_hash = int(desc["StreamDescription"]["Shards"][0]["HashKeyRange"]["EndingHashKey"])
mid = str((start_hash + end_hash) // 2)
kin.split_shard(StreamName=sname, ShardToSplit=shard_id, NewStartingHashKey=mid)
time.sleep(0.3)
desc2 = kin.describe_stream(StreamName=sname)
assert len(desc2["StreamDescription"]["Shards"]) == 2
kin.delete_stream(StreamName=sname)
def test_kinesis_merge_shards(kin):
import uuid as _uuid
sname = f"intg-merge-{_uuid.uuid4().hex[:8]}"
kin.create_stream(StreamName=sname, ShardCount=2)
time.sleep(0.3)
desc = kin.describe_stream(StreamName=sname)
shards = desc["StreamDescription"]["Shards"]
assert len(shards) == 2
# Sort by starting hash key to get adjacent shards
shards_sorted = sorted(shards, key=lambda s: int(s["HashKeyRange"]["StartingHashKey"]))
kin.merge_shards(
StreamName=sname,
ShardToMerge=shards_sorted[0]["ShardId"],
AdjacentShardToMerge=shards_sorted[1]["ShardId"],
)
time.sleep(0.3)
desc2 = kin.describe_stream(StreamName=sname)
assert len(desc2["StreamDescription"]["Shards"]) == 1
kin.delete_stream(StreamName=sname)
def test_kinesis_update_shard_count(kin):
import uuid as _uuid
sname = f"intg-usc-{_uuid.uuid4().hex[:8]}"
kin.create_stream(StreamName=sname, ShardCount=1)
time.sleep(0.3)
resp = kin.update_shard_count(StreamName=sname, TargetShardCount=2, ScalingType="UNIFORM_SCALING")
assert resp["TargetShardCount"] == 2
kin.delete_stream(StreamName=sname)
def test_kinesis_register_deregister_consumer(kin):
import uuid as _uuid
sname = f"intg-consumer-{_uuid.uuid4().hex[:8]}"
kin.create_stream(StreamName=sname, ShardCount=1)
time.sleep(0.3)
desc = kin.describe_stream(StreamName=sname)
stream_arn = desc["StreamDescription"]["StreamARN"]
resp = kin.register_stream_consumer(StreamARN=stream_arn, ConsumerName="my-consumer")
assert resp["Consumer"]["ConsumerName"] == "my-consumer"
assert resp["Consumer"]["ConsumerStatus"] == "ACTIVE"
consumer_arn = resp["Consumer"]["ConsumerARN"]
consumers = kin.list_stream_consumers(StreamARN=stream_arn)
assert any(c["ConsumerName"] == "my-consumer" for c in consumers["Consumers"])
desc_c = kin.describe_stream_consumer(ConsumerARN=consumer_arn)
assert desc_c["ConsumerDescription"]["ConsumerName"] == "my-consumer"
kin.deregister_stream_consumer(ConsumerARN=consumer_arn)
consumers2 = kin.list_stream_consumers(StreamARN=stream_arn)
assert not any(c["ConsumerName"] == "my-consumer" for c in consumers2["Consumers"])
kin.delete_stream(StreamName=sname)
def _subscribe_setup(kin, shards=1):
sname = f"intg-sub-{_uuid_mod.uuid4().hex[:8]}"
kin.create_stream(StreamName=sname, ShardCount=shards)
stream_arn = kin.describe_stream(StreamName=sname)["StreamDescription"]["StreamARN"]
consumer_arn = kin.register_stream_consumer(
StreamARN=stream_arn, ConsumerName="efo")["Consumer"]["ConsumerARN"]
return sname, consumer_arn
def test_kinesis_subscribe_to_shard_pushes_backlog_then_new_records(kin):
sname, consumer_arn = _subscribe_setup(kin)
first = [kin.put_record(StreamName=sname, Data=f"r{i}".encode(), PartitionKey="k")["SequenceNumber"]
for i in range(2)]
resp = kin.subscribe_to_shard(ConsumerARN=consumer_arn, ShardId="shardId-000000000000",
StartingPosition={"Type": "TRIM_HORIZON"})
events = iter(resp["EventStream"])
event = next(events)["SubscribeToShardEvent"]
assert [r["Data"] for r in event["Records"]] == [b"r0", b"r1"]
assert [r["SequenceNumber"] for r in event["Records"]] == first
assert event["ContinuationSequenceNumber"] == first[-1]
assert event["MillisBehindLatest"] == 0
third = kin.put_record(StreamName=sname, Data=b"r2", PartitionKey="k")["SequenceNumber"]
event = next(events)["SubscribeToShardEvent"]
assert [(r["Data"], r["PartitionKey"]) for r in event["Records"]] == [(b"r2", "k")]
assert event["ContinuationSequenceNumber"] == third
resp["EventStream"].close()
resumed = kin.subscribe_to_shard(
ConsumerARN=consumer_arn, ShardId="shardId-000000000000",
StartingPosition={"Type": "AFTER_SEQUENCE_NUMBER", "SequenceNumber": first[0]})
with pytest.raises(ClientError) as exc: # refused within 5 s of the first call
kin.subscribe_to_shard(ConsumerARN=consumer_arn, ShardId="shardId-000000000000",
StartingPosition={"Type": "LATEST"})
assert exc.value.response["Error"]["Code"] == "ResourceInUseException"
event = next(iter(resumed["EventStream"]))["SubscribeToShardEvent"]
assert [r["SequenceNumber"] for r in event["Records"]] == [first[1], third]
resumed["EventStream"].close()
kin.delete_stream(StreamName=sname)
def test_kinesis_subscribe_to_shard_latest_starts_empty_and_takeover_ends_the_old_stream(kin):
sname, consumer_arn = _subscribe_setup(kin)
kin.put_record(StreamName=sname, Data=b"old", PartitionKey="k")
old = kin.subscribe_to_shard(ConsumerARN=consumer_arn, ShardId="shardId-000000000000",
StartingPosition={"Type": "LATEST"})
old_events = iter(old["EventStream"])
event = next(old_events)["SubscribeToShardEvent"]
assert event["Records"] == [] and event["ContinuationSequenceNumber"]
time.sleep(5.2)
new = kin.subscribe_to_shard(ConsumerARN=consumer_arn, ShardId="shardId-000000000000",
StartingPosition={"Type": "LATEST"})
with pytest.raises(ClientError) as exc:
for _ in old_events:
pass
assert exc.value.response["Error"]["Code"] == "ResourceInUseException"
assert next(iter(new["EventStream"]))["SubscribeToShardEvent"]["Records"] == []
new["EventStream"].close()
kin.delete_stream(StreamName=sname)
def test_kinesis_subscribe_to_shard_ends_with_child_shards_after_a_split(kin):
sname, consumer_arn = _subscribe_setup(kin)
resp = kin.subscribe_to_shard(ConsumerARN=consumer_arn, ShardId="shardId-000000000000",
StartingPosition={"Type": "TRIM_HORIZON"})
events = iter(resp["EventStream"])
assert "ChildShards" not in next(events)["SubscribeToShardEvent"]
kin.split_shard(StreamName=sname, ShardToSplit="shardId-000000000000",
NewStartingHashKey=str(2**127))
last = [e["SubscribeToShardEvent"] for e in events][-1]
assert sorted(c["ShardId"] for c in last["ChildShards"]) == [
"shardId-000000000001", "shardId-000000000002"]
assert all(c["ParentShards"] == ["shardId-000000000000"] for c in last["ChildShards"])
kin.delete_stream(StreamName=sname)
def test_kinesis_subscribe_to_shard_errors(kin):
sname, consumer_arn = _subscribe_setup(kin)
cases = [
(dict(ConsumerARN=consumer_arn + "0", ShardId="shardId-000000000000",
StartingPosition={"Type": "LATEST"}), "ResourceNotFoundException"),
(dict(ConsumerARN=consumer_arn, ShardId="shardId-000000000009",
StartingPosition={"Type": "LATEST"}), "ResourceNotFoundException"),
(dict(ConsumerARN=consumer_arn, ShardId="shardId-000000000000",
StartingPosition={"Type": "AT_SEQUENCE_NUMBER"}), "InvalidArgumentException"),
]
for kwargs, code in cases:
with pytest.raises(ClientError) as exc:
kin.subscribe_to_shard(**kwargs)
assert exc.value.response["Error"]["Code"] == code
kin.delete_stream(StreamName=sname)
def test_kinesis_subscribe_to_shard_cbor_events(kin):
import urllib.request
import cbor2
from botocore.eventstream import EventStreamBuffer
sname, consumer_arn = _subscribe_setup(kin)
kin.put_record(StreamName=sname, Data=b"\x00\x01", PartitionKey="k")
request = urllib.request.Request(
kin.meta.endpoint_url, method="POST",
data=cbor2.dumps({"ConsumerARN": consumer_arn, "ShardId": "shardId-000000000000",
"StartingPosition": {"Type": "TRIM_HORIZON"}}),
headers={"Content-Type": "application/x-amz-cbor-1.1",
"X-Amz-Target": "Kinesis_20131202.SubscribeToShard",
"Authorization": "AWS4-HMAC-SHA256 Credential=test/20260101/us-east-1/kinesis/aws4_request"})
buffer, messages = EventStreamBuffer(), []
with urllib.request.urlopen(request, timeout=10) as response:
while len(messages) < 2:
buffer.add_data(response.read1(65536))
messages.extend(buffer)
assert messages[0].headers[":event-type"] == "initial-response"
event = messages[1]
assert event.headers[":content-type"] == "application/x-amz-cbor-1.1"
record = cbor2.loads(event.payload)["Records"][0]
assert record["Data"] == b"\x00\x01"
kin.delete_stream(StreamName=sname)
def test_kinesis_consumer_arns_reject_foreign_region_without_exact_key_fallback(kin):
"""ConsumerARN paths must not resolve a consumer owned by another request region."""
west = _regional_kin("us-west-2")
stream_name = f"kin-consumer-arn-scope-{_uuid_mod.uuid4().hex[:8]}"
consumer_name = "west-consumer"
west.create_stream(StreamName=stream_name, ShardCount=1)
west_arn = west.describe_stream(StreamName=stream_name)["StreamDescription"]["StreamARN"]
consumer_arn = west.register_stream_consumer(
StreamARN=west_arn,
ConsumerName=consumer_name,
)["Consumer"]["ConsumerARN"]
with pytest.raises(ClientError) as exc:
kin.list_stream_consumers(StreamARN=west_arn)
assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
for kwargs in (
{"ConsumerARN": consumer_arn},
{"StreamARN": west_arn, "ConsumerName": consumer_name},
):
with pytest.raises(ClientError) as exc:
kin.describe_stream_consumer(**kwargs)
assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
with pytest.raises(ClientError) as exc:
kin.deregister_stream_consumer(ConsumerARN=consumer_arn)
assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
assert west.describe_stream_consumer(ConsumerARN=consumer_arn)["ConsumerDescription"]["ConsumerName"] == consumer_name
def test_kinesis_restore_legacy_account_scoped_state_uses_arn_region():
from ministack.core.responses import (
AccountScopedDict,
get_account_id,
get_region,
set_request_account_id,
set_request_region,
)
from ministack.services import kinesis as _kin
original_account = get_account_id()
original_region = get_region()
account_id = "000000000000"
stream_name = "LegacyKinesisStream"
stream_arn = f"arn:aws:kinesis:us-west-2:{account_id}:stream/{stream_name}"
consumer_arn = f"{stream_arn}/consumer/legacy-consumer:1700000000"
legacy_streams = AccountScopedDict()
legacy_streams._data[(account_id, stream_name)] = {
"StreamName": stream_name,
"StreamARN": stream_arn,
"StreamStatus": "ACTIVE",
"StreamModeDetails": {"StreamMode": "PROVISIONED"},
"RetentionPeriodHours": 24,
"shards": {},
"tags": {},
"CreationTimestamp": 1700000000,
"EncryptionType": "NONE",
}
legacy_iterators = AccountScopedDict()
legacy_iterators._data[(account_id, "legacy-iterator-with-arn")] = {
"stream": stream_name,
"stream_arn": stream_arn,
"shard_id": "shardId-000000000000",
"position": 0,
"created_at": 1700000000,
}
legacy_iterators._data[(account_id, "legacy-iterator-with-stream-name")] = {
"stream": stream_name,
"shard_id": "shardId-000000000000",
"position": 0,
"created_at": 1700000000,
}
legacy_consumers = AccountScopedDict()
legacy_consumers._data[(account_id, consumer_arn)] = {
"ConsumerARN": consumer_arn,
"ConsumerName": "legacy-consumer",
"ConsumerStatus": "ACTIVE",
"ConsumerCreationTimestamp": 1700000000,
"StreamARN": stream_arn,
}
_kin.reset()
try:
set_request_account_id(account_id)
set_request_region("us-east-1")
_kin.load_persisted_state({
"streams": legacy_streams,
"shard_iterators": legacy_iterators,
"consumers": legacy_consumers,
})
assert _kin._streams.get_scoped(account_id, "us-east-1", stream_name) is None
assert _kin._streams.get_scoped(account_id, "us-west-2", stream_name)["StreamARN"] == stream_arn
assert _kin._shard_iterators.get_scoped(
account_id, "us-west-2", "legacy-iterator-with-arn"
)["stream_arn"] == stream_arn
stream_name_iterator = _kin._shard_iterators.get_scoped(
account_id, "us-west-2", "legacy-iterator-with-stream-name"
)
assert stream_name_iterator["stream"] == stream_name
assert "stream_arn" not in stream_name_iterator
assert _kin._consumers.get_scoped(account_id, "us-west-2", consumer_arn)["ConsumerARN"] == consumer_arn
finally:
_kin.reset()
set_request_account_id(original_account)
set_request_region(original_region)
def test_kinesis_at_timestamp_iterator(kin):
"""AT_TIMESTAMP shard iterator returns records after the given timestamp."""
kin.create_stream(StreamName="qa-kin-ts", ShardCount=1)
time.sleep(0.1)
before = time.time()
kin.put_record(StreamName="qa-kin-ts", Data=b"after-ts", PartitionKey="pk")
shards = kin.describe_stream(StreamName="qa-kin-ts")["StreamDescription"]["Shards"]
shard_id = shards[0]["ShardId"]
it = kin.get_shard_iterator(
StreamName="qa-kin-ts",
ShardId=shard_id,
ShardIteratorType="AT_TIMESTAMP",
Timestamp=before,
)["ShardIterator"]
records = kin.get_records(ShardIterator=it, Limit=10)["Records"]
assert len(records) >= 1
assert any(r["Data"] == b"after-ts" for r in records)
def test_kinesis_retention_period(kin):
"""IncreaseStreamRetentionPeriod / DecreaseStreamRetentionPeriod."""
kin.create_stream(StreamName="qa-kin-retention", ShardCount=1)
kin.increase_stream_retention_period(StreamName="qa-kin-retention", RetentionPeriodHours=48)
desc = kin.describe_stream(StreamName="qa-kin-retention")["StreamDescription"]
assert desc["RetentionPeriodHours"] == 48
kin.decrease_stream_retention_period(StreamName="qa-kin-retention", RetentionPeriodHours=24)
desc2 = kin.describe_stream(StreamName="qa-kin-retention")["StreamDescription"]
assert desc2["RetentionPeriodHours"] == 24
def test_kinesis_stream_encryption_toggle(kin):
"""StartStreamEncryption / StopStreamEncryption."""
kin.create_stream(StreamName="qa-kin-enc", ShardCount=1)
kin.start_stream_encryption(
StreamName="qa-kin-enc",
EncryptionType="KMS",
KeyId="alias/aws/kinesis",
)
desc = kin.describe_stream(StreamName="qa-kin-enc")["StreamDescription"]
assert desc["EncryptionType"] == "KMS"
kin.stop_stream_encryption(
StreamName="qa-kin-enc",
EncryptionType="KMS",
KeyId="alias/aws/kinesis",
)
desc2 = kin.describe_stream(StreamName="qa-kin-enc")["StreamDescription"]
assert desc2["EncryptionType"] == "NONE"
def test_kinesis_put_record_oversized(kin):
kin.create_stream(StreamName="kin-limits", ShardCount=1)
from botocore.exceptions import ClientError
with pytest.raises(ClientError) as exc:
kin.put_record(StreamName="kin-limits", Data=b"x" * (1024 * 1024 + 1), PartitionKey="pk")
assert "1048576" in str(exc.value)
def test_kinesis_put_record_partition_key_too_long(kin):
from botocore.exceptions import ClientError
with pytest.raises(ClientError) as exc:
kin.put_record(StreamName="kin-limits", Data=b"ok", PartitionKey="k" * 257)
assert "256" in str(exc.value)
def test_kinesis_put_records_batch_over_500(kin):
from botocore.exceptions import ClientError
with pytest.raises(ClientError) as exc:
kin.put_records(
StreamName="kin-limits",
Records=[{"Data": b"x", "PartitionKey": "pk"} for _ in range(501)],
)
assert "500" in str(exc.value)
def test_kinesis_put_records_total_payload_over_5mb(kin):
from botocore.exceptions import ClientError
# 6 records of ~1MB each = ~6MB > 5MB limit
with pytest.raises(ClientError) as exc:
kin.put_records(
StreamName="kin-limits",
Records=[{"Data": b"x" * (1024 * 1024), "PartitionKey": "pk"} for _ in range(6)],
)
assert "5 MB" in str(exc.value)
def test_kinesis_esm_creates_and_lists(lam, kin):
"""Kinesis ESM can be created and listed."""
kin.create_stream(StreamName="esm-kin-stream", ShardCount=1)
stream = kin.describe_stream(StreamName="esm-kin-stream")["StreamDescription"]
stream_arn = stream["StreamARN"]
code = "def handler(event, context): return len(event.get('Records', []))"
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
zf.writestr("index.py", code)
lam.create_function(
FunctionName="esm-kin-fn", Runtime="python3.11",
Role=_LAMBDA_ROLE, Handler="index.handler",
Code={"ZipFile": buf.getvalue()},
)
esm = lam.create_event_source_mapping(
FunctionName="esm-kin-fn",
EventSourceArn=stream_arn,
StartingPosition="TRIM_HORIZON",
BatchSize=10,
)
assert esm["EventSourceArn"] == stream_arn
assert esm["FunctionArn"].endswith("esm-kin-fn")
esms = lam.list_event_source_mappings(FunctionName="esm-kin-fn")["EventSourceMappings"]
assert any(e["UUID"] == esm["UUID"] for e in esms)
lam.delete_event_source_mapping(UUID=esm["UUID"])
lam.delete_function(FunctionName="esm-kin-fn")
def test_kinesis_iterator_reuse_on_retry(kin):
"""Same shard iterator can be used multiple times (client retry), matching AWS behavior."""
kin.create_stream(StreamName="kin-iter-retry", ShardCount=1)
kin.put_record(StreamName="kin-iter-retry", Data=b"rec1", PartitionKey="pk1")
kin.put_record(StreamName="kin-iter-retry", Data=b"rec2", PartitionKey="pk2")
desc = kin.describe_stream(StreamName="kin-iter-retry")
shard_id = desc["StreamDescription"]["Shards"][0]["ShardId"]
it = kin.get_shard_iterator(
StreamName="kin-iter-retry", ShardId=shard_id, ShardIteratorType="TRIM_HORIZON"
)["ShardIterator"]
# First call with iterator
resp1 = kin.get_records(ShardIterator=it)
assert len(resp1["Records"]) == 2
# Retry with the same iterator — should succeed and return identical data
resp2 = kin.get_records(ShardIterator=it)
assert len(resp2["Records"]) == 2
assert resp2["Records"][0]["Data"] == resp1["Records"][0]["Data"]
# NextShardIterator from first call should advance past existing records
resp3 = kin.get_records(ShardIterator=resp1["NextShardIterator"])
assert len(resp3["Records"]) == 0
def test_kinesis_cbor_put_record(kin):
"""Java SDK sends CBOR-encoded PutRecord; ministack must decode it."""
import urllib.request
import cbor2
endpoint = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566")
kin.create_stream(StreamName="cbor-test-stream", ShardCount=1)
# Build a CBOR-encoded PutRecord payload (same as AWS Java SDK v2 sends)
cbor_body = cbor2.dumps({
"StreamName": "cbor-test-stream",
"Data": b'{ "test": "123"}',
"PartitionKey": "1",
})
req = urllib.request.Request(
endpoint,
data=cbor_body,
headers={
"Content-Type": "application/x-amz-cbor-1.1",
"X-Amz-Target": "Kinesis_20131202.PutRecord",
},
method="POST",
)
with urllib.request.urlopen(req) as resp:
assert resp.status == 200
resp_body = cbor2.loads(resp.read())
assert "ShardId" in resp_body
assert "SequenceNumber" in resp_body
# Verify the record is retrievable via normal JSON path
desc = kin.describe_stream(StreamName="cbor-test-stream")
shard_id = desc["StreamDescription"]["Shards"][0]["ShardId"]
it = kin.get_shard_iterator(
StreamName="cbor-test-stream", ShardId=shard_id, ShardIteratorType="TRIM_HORIZON"
)
records = kin.get_records(ShardIterator=it["ShardIterator"])
assert len(records["Records"]) == 1
def test_kinesis_describe_echoes_the_encryption_key_id(kin):
"""StartStreamEncryption stores KeyId; the describe paths must return it.
AWS reports KeyId alongside EncryptionType in both StreamDescription and
StreamDescriptionSummary. Storing it without echoing it means the Terraform
provider reads kms_key_id as empty on a stream that is in fact encrypted, so
every plan proposes setting it again.
"""
name = "qa-kinesis-keyid"
kin.create_stream(StreamName=name, ShardCount=1)
key = "arn:aws:kms:us-east-1:000000000000:alias/aws/kinesis"
kin.start_stream_encryption(
StreamName=name, EncryptionType="KMS", KeyId=key
)
desc = kin.describe_stream(StreamName=name)["StreamDescription"]
assert desc["EncryptionType"] == "KMS"
assert desc.get("KeyId") == key
summary = kin.describe_stream_summary(StreamName=name)[
"StreamDescriptionSummary"
]
assert summary["EncryptionType"] == "KMS"
assert summary.get("KeyId") == key
# An unencrypted stream must not report a KeyId at all — AWS omits the field
# rather than returning an empty one, and an empty string would read as drift.
kin.stop_stream_encryption(
StreamName=name, EncryptionType="KMS", KeyId=key
)
desc = kin.describe_stream(StreamName=name)["StreamDescription"]
assert desc["EncryptionType"] == "NONE"
assert "KeyId" not in desc
# --- SubscribeToShard across retention pruning (in-process dispatch) ---
_SUB_STARTS = ("TRIM_HORIZON", "LATEST", "AT_SEQUENCE_NUMBER", "AFTER_SEQUENCE_NUMBER", "AT_TIMESTAMP")
_SUB_SHARD = "shardId-000000000000"
@pytest.fixture
def subscription_state(monkeypatch):
clock = [1_700_000_000.0]
monkeypatch.setattr(kinesis, "time", SimpleNamespace(time=lambda: clock[0]))
monkeypatch.setattr(kinesis, "_streams", AccountRegionScopedDict())
monkeypatch.setattr(kinesis, "_consumers", AccountRegionScopedDict())
monkeypatch.setattr(kinesis, "_subscriptions", {})
monkeypatch.setattr(kinesis, "_SUBSCRIPTION_IDLE_SECONDS", 0)
return clock
async def _sub_call(action, data, is_cbor=False):
encode = cbor2.dumps if is_cbor else lambda value: json.dumps(value).encode()
headers = {"x-amz-target": f"Kinesis_20131202.{action}"}
if is_cbor:
headers["content-type"] = "application/x-amz-cbor-1.1"
result = await kinesis.handle_request("POST", "/", headers, encode(data), {})
assert result[0] == 200, result
if action == "SubscribeToShard":
return result[2]
return cbor2.loads(result[2]) if is_cbor else json.loads(result[2])
async def _sub_setup():
await _sub_call("CreateStream", {"StreamName": "retention", "ShardCount": 1})
arn = (await _sub_call("DescribeStream", {"StreamName": "retention"}))["StreamDescription"]["StreamARN"]
consumer = await _sub_call("RegisterStreamConsumer", {"StreamARN": arn, "ConsumerName": "reader"})
return consumer["Consumer"]["ConsumerARN"]
async def _sub_put(value):
return (await _sub_call("PutRecord", {
"StreamName": "retention", "PartitionKey": "p", "Data": base64.b64encode(value).decode(),
}))["SequenceNumber"]
def _sub_starting(kind, sequence, timestamp):
result = {"Type": kind}
if "SEQUENCE_NUMBER" in kind:
result["SequenceNumber"] = sequence
if kind == "AT_TIMESTAMP":
result["Timestamp"] = timestamp
return result
async def _sub_run(response, is_cbor, on_event):
disconnect = asyncio.Event()
events = []
async def receive():
await disconnect.wait()
return {"type": "http.disconnect"}
async def send(message):
if not message.get("body"):
return
buffer = EventStreamBuffer()
buffer.add_data(message["body"])
for frame in buffer:
assert frame.headers[":message-type"] == "event", frame.headers
assert frame.headers[":content-type"] == (
"application/x-amz-cbor-1.1" if is_cbor else "application/json")
if frame.headers[":event-type"] == "initial-response":
continue
assert frame.headers[":event-type"] == "SubscribeToShardEvent"
event = cbor2.loads(frame.payload) if is_cbor else json.loads(frame.payload)
events.append(event)
if await on_event(event, len(events)):
disconnect.set()
await asyncio.wait_for(response.runner(send, receive), timeout=5)
return events
@pytest.mark.parametrize("is_cbor", [False, True], ids=["json", "cbor"])
@pytest.mark.parametrize("kind", _SUB_STARTS)
def test_subscription_delivers_after_repeated_retention_and_idle_polls(subscription_state, kind, is_cbor):
clock = subscription_state
async def exercise():
with request_scope("000000000000", "us-east-1"):
consumer = await _sub_setup()
old = await _sub_put(b"old")
timestamp = clock[0]
clock[0] += 3
retained = await _sub_put(b"retained")
clock[0] += 24 * 3600 - 4
response = await _sub_call("SubscribeToShard", {
"ConsumerARN": consumer, "ShardId": _SUB_SHARD,
"StartingPosition": _sub_starting(kind, old, timestamp),
}, is_cbor)
expected = []
async def on_event(event, number):
# The two backlog records expire separately within the normal
# five-minute subscription lifetime. Idle and append after
# each pruning, while keeping newly delivered records alive.
if number == 1:
clock[0] += 2
kinesis._expire_records(kinesis._streams["retention"])
elif number == 4:
clock[0] += 3
kinesis._expire_records(kinesis._streams["retention"])
elif number in (2, 5):
expected.append(await _sub_put(f"new-{number}".encode()))
return number == 7
events = await _sub_run(response, is_cbor, on_event)
initial = ([] if kind == "LATEST" else [retained] if kind == "AFTER_SEQUENCE_NUMBER"
else [old, retained])
assert [r["SequenceNumber"] for r in events[0]["Records"]] == initial
assert [r["SequenceNumber"] for e in events[1:] for r in e["Records"]] == expected
assert events[1]["Records"] == events[4]["Records"] == events[6]["Records"] == []
assert events[1]["ContinuationSequenceNumber"] == events[0]["ContinuationSequenceNumber"]
assert events[2]["ContinuationSequenceNumber"] == expected[0]
assert events[3]["ContinuationSequenceNumber"] == expected[0]
assert events[4]["ContinuationSequenceNumber"] == expected[0]
assert events[5]["ContinuationSequenceNumber"] == expected[1]
assert events[6]["ContinuationSequenceNumber"] == expected[1]
assert all(e["MillisBehindLatest"] == 0 and "ChildShards" not in e for e in events)
records = events[2]["Records"] + events[5]["Records"]
assert [r["Data"] if is_cbor else base64.b64decode(r["Data"]) for r in records] == [
b"new-2", b"new-5"]
assert not kinesis._subscriptions
asyncio.run(exercise())
@pytest.mark.parametrize("is_cbor", [False, True], ids=["json", "cbor"])
@pytest.mark.parametrize("kind", _SUB_STARTS)
def test_subscription_keeps_checkpoint_when_all_records_expire_before_runner(subscription_state, kind, is_cbor):
clock = subscription_state
async def exercise():
with request_scope("000000000000", "us-east-1"):
consumer = await _sub_setup()
old = await _sub_put(b"old")
timestamp = clock[0]
shard_start = kinesis._streams["retention"]["shards"][_SUB_SHARD]["starting_sequence_number"]
clock[0] += 24 * 3600 - 1
response = await _sub_call("SubscribeToShard", {
"ConsumerARN": consumer, "ShardId": _SUB_SHARD,
"StartingPosition": _sub_starting(kind, old, timestamp),
}, is_cbor)
clock[0] += 2
kinesis._expire_records(kinesis._streams["retention"])
async def on_event(event, number):
return True
events = await _sub_run(response, is_cbor, on_event)
assert events[0]["Records"] == []
# Preserve the existing selected checkpoint value; do not invent
# an AWS empty-shard checkpoint number.
checkpoint = old if kind in ("LATEST", "AFTER_SEQUENCE_NUMBER") else shard_start
assert events[0]["ContinuationSequenceNumber"] == checkpoint
asyncio.run(exercise())
@pytest.mark.parametrize("is_cbor", [False, True], ids=["json", "cbor"])
@pytest.mark.parametrize("kind", _SUB_STARTS)
def test_subscription_captures_selection_before_runner_and_pruning(subscription_state, kind, is_cbor):
clock = subscription_state
async def exercise():
with request_scope("000000000000", "us-east-1"):
consumer = await _sub_setup()
await _sub_put(b"expired-prefix")
clock[0] += 1
selected = await _sub_put(b"selected")
timestamp = clock[0]
clock[0] += 24 * 3600 - 1
response = await _sub_call("SubscribeToShard", {
"ConsumerARN": consumer, "ShardId": _SUB_SHARD,
"StartingPosition": _sub_starting(kind, selected, timestamp),
}, is_cbor)
# Only the prefix expires. Append between response creation and
# runner startup, so LATEST must retain the establishment boundary.
clock[0] += 1
appended = await _sub_put(b"appended")
async def on_event(event, number):
return True
# Run with another request's context to exercise the saved tenant.
with request_scope("111111111111", "eu-west-1"):
events = await _sub_run(response, is_cbor, on_event)
expected = [appended] if kind in ("LATEST", "AFTER_SEQUENCE_NUMBER") else [selected, appended]
assert [r["SequenceNumber"] for r in events[0]["Records"]] == expected
assert events[0]["ContinuationSequenceNumber"] == appended
asyncio.run(exercise())
def _kin_error(exc):
err = exc.value.response["Error"]
return exc.value.response["ResponseMetadata"]["HTTPStatusCode"], err["Code"], err["Message"]
def test_kinesis_resource_policy_follows_aws(kin):
"""Captured on AWS (us-east-1, 2026-10-26): codes, messages and check order of the resource-policy calls."""
name = f"rp-{_uuid_mod.uuid4().hex[:8]}"
kin.create_stream(StreamName=name, ShardCount=1)
arn = kin.describe_stream_summary(StreamName=name)["StreamDescriptionSummary"]["StreamARN"]
account = arn.split(":")[4]
consumer = kin.register_stream_consumer(StreamARN=arn, ConsumerName="c")["Consumer"]["ConsumerARN"]
def policy(resource, actions):
return json.dumps({"Version": "2012-10-17", "Statement": [{"Sid": "x", "Effect": "Allow",
"Principal": {"AWS": f"arn:aws:iam::{account}:root"}, "Action": actions, "Resource": resource}]})
try:
assert kin.get_resource_policy(ResourceARN=arn)["Policy"] == "{}"
with pytest.raises(ClientError) as exc:
kin.delete_resource_policy(ResourceARN=arn)
assert _kin_error(exc) == (400, "ResourceNotFoundException", f"No resource policy found for resource ARN {arn}.")
for document, message in (
("not json", "Policy validation error: This policy contains invalid Json"),
("{}", "Policy validation error: Missing required field Version"),
('{"Version":"2012-10-17"}', "Policy validation error: Missing required field Statement"),
(policy(consumer, ["kinesis:GetRecords"]), "Policy validation error: Resource policy contains invalid statement"
" at line 1, column 41: The resource policy's resource must be the same as the resource ARN."),
):
with pytest.raises(ClientError) as exc:
kin.put_resource_policy(ResourceARN=arn, Policy=document)
assert _kin_error(exc) == (400, "InvalidArgumentException", message)
with pytest.raises(ClientError) as exc:
kin.put_resource_policy(ResourceARN=consumer, Policy=policy(consumer, ["kinesis:GetRecords"]))
assert _kin_error(exc) == (400, "InvalidArgumentException", "Policy validation error: Resource policy contains invalid"
' statement at line 1, column 41: The following action names are invalid: "kinesis:GetRecords"')
stream_policy = policy(arn, ["kinesis:GetRecords", "kinesis:GetShardIterator"])
kin.put_resource_policy(ResourceARN=arn, Policy=stream_policy)
assert kin.get_resource_policy(ResourceARN=arn)["Policy"] == stream_policy
kin.put_resource_policy(ResourceARN=consumer, Policy=policy(consumer, ["kinesis:SubscribeToShard"]))
kin.delete_resource_policy(ResourceARN=arn)
assert kin.get_resource_policy(ResourceARN=arn)["Policy"] == "{}"
missing = arn.rsplit("/", 1)[0] + "/no-such-stream"
with pytest.raises(ClientError) as exc:
kin.get_resource_policy(ResourceARN=missing)
assert _kin_error(exc) == (400, "ResourceNotFoundException", f"Stream no-such-stream under account {account} not found.")
with pytest.raises(ClientError) as exc:
kin.get_resource_policy(ResourceARN=arn.replace(f":{account}:", ":111111111111:"))
assert _kin_error(exc) == (400, "UnrecognizedClientException", "No account found for the given parameters")
west = arn.replace(":us-east-1:", ":us-west-2:")
with pytest.raises(ClientError) as exc:
kin.get_resource_policy(ResourceARN=west)
assert _kin_error(exc) == (400, "InvalidArgumentException",
f"The region specified in the ARN '{west}' does not match the endpoint region")
finally:
kin.delete_stream(StreamName=name, EnforceConsumerDeletion=True)
def test_kinesis_resource_policy_goes_with_its_stream(kin):
name = f"rp-del-{_uuid_mod.uuid4().hex[:8]}"
kin.create_stream(StreamName=name, ShardCount=1)
arn = kin.describe_stream_summary(StreamName=name)["StreamDescriptionSummary"]["StreamARN"]
account = arn.split(":")[4]
kin.put_resource_policy(ResourceARN=arn, Policy=json.dumps({"Version": "2012-10-17", "Statement": [{
"Effect": "Allow", "Principal": {"AWS": f"arn:aws:iam::{account}:root"}, "Action": "kinesis:GetRecords",
"Resource": arn}]}))
kin.delete_stream(StreamName=name)
kin.create_stream(StreamName=name, ShardCount=1)
try:
assert kin.get_resource_policy(ResourceARN=arn)["Policy"] == "{}"
finally:
kin.delete_stream(StreamName=name)
_FILTER_HANDLER = """
import base64
def handler(event, context):
for r in event["Records"]:
print("XREC " + base64.b64decode(r["kinesis"]["data"]).decode())
return len(event["Records"])
"""
@pytest.mark.parametrize("pattern, expected", [
({"data": {"kind": ["keep"]}}, ['{"kind": "keep"}', '{"kind": "keep", "n": 2}']),
({"data": {"n": [{"numeric": [">", 1]}]}}, ['{"kind": "keep", "n": 2}']),
({"partitionKey": ["pk-b"]}, ['{"kind": "drop"}', "not json"]),
({"partitionKey": [{"prefix": "pk-"}], "data": {"kind": [{"exists": True}]}},
['{"kind": "keep"}', '{"kind": "drop"}', '{"kind": "keep", "n": 2}']),
], ids=["data-field", "data-numeric", "metadata-only-matches-non-json", "metadata-and-data"])
def test_kinesis_esm_filter_criteria_match_decoded_data_and_metadata(lam, kin, logs, pattern, expected):
"""Black box: the function only receives the records the pattern selects."""
stream = f"kin-filter-{_uuid_mod.uuid4().hex[:8]}"
kin.create_stream(StreamName=stream, ShardCount=1)
arn = kin.describe_stream_summary(StreamName=stream)["StreamDescriptionSummary"]["StreamARN"]
for data, pk in (('{"kind": "keep"}', "pk-a"), ('{"kind": "drop"}', "pk-b"),
("not json", "pk-b"), ('{"kind": "keep", "n": 2}', "pk-a")):
kin.put_record(StreamName=stream, Data=data.encode(), PartitionKey=pk)
fn = f"kin-filter-fn-{_uuid_mod.uuid4().hex[:8]}"
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
zf.writestr("index.py", _FILTER_HANDLER)
lam.create_function(FunctionName=fn, Runtime="python3.12", Role=_LAMBDA_ROLE,
Handler="index.handler", Code={"ZipFile": buf.getvalue()})
esm = lam.create_event_source_mapping(
FunctionName=fn, EventSourceArn=arn, StartingPosition="TRIM_HORIZON",
FilterCriteria={"Filters": [{"Pattern": json.dumps(pattern)}]})["UUID"]
try:
received = []
deadline = time.time() + 30
while time.time() < deadline:
try:
events = logs.filter_log_events(logGroupName=f"/aws/lambda/{fn}")["events"]
except ClientError:
events = []
received = [line[len("XREC "):] for e in events for line in e["message"].splitlines()
if line.startswith("XREC ")]
if len(received) >= len(expected):
break
time.sleep(0.5)
time.sleep(2) # nothing that should have been filtered turns up later
events = logs.filter_log_events(logGroupName=f"/aws/lambda/{fn}")["events"]
received = [line[len("XREC "):] for e in events for line in e["message"].splitlines()
if line.startswith("XREC ")]
assert received == expected
finally:
lam.delete_event_source_mapping(UUID=esm)
lam.delete_function(FunctionName=fn)
kin.delete_stream(StreamName=stream)
def _account_client(service, account):
return boto3.client(service, endpoint_url=os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566"),
aws_access_key_id=account, aws_secret_access_key="test", region_name="us-east-1")
def test_kinesis_esm_reads_a_stream_in_another_account(lam, logs, auth_enabled):
"""A function in one account maps a stream in another (same region); records reach it. Under AUTH=true
the stream's resource policy must allow the function's execution role."""
if auth_enabled:
pytest.skip("account selection by access key needs AUTH unset")
owner = _account_client("kinesis", "111111111111")
stream = f"xacct-{_uuid_mod.uuid4().hex[:8]}"
owner.create_stream(StreamName=stream, ShardCount=1)
arn = owner.describe_stream_summary(StreamName=stream)["StreamDescriptionSummary"]["StreamARN"]
for n in range(3):
owner.put_record(StreamName=stream, Data=f"rec-{n}".encode(), PartitionKey="pk")
fn = f"xacct-fn-{_uuid_mod.uuid4().hex[:8]}"
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
zf.writestr("index.py", _FILTER_HANDLER)
lam.create_function(FunctionName=fn, Runtime="python3.12", Role=_LAMBDA_ROLE,
Handler="index.handler", Code={"ZipFile": buf.getvalue()})
try:
missing = arn.rsplit("/", 1)[0] + "/no-such-stream"
with pytest.raises(ClientError) as exc:
lam.create_event_source_mapping(FunctionName=fn, EventSourceArn=missing, StartingPosition="LATEST")
assert exc.value.response["Error"]["Message"] == (
"Received Exception while reading from provided stream. "
"Stream no-such-stream under account 111111111111 not found.")
esm = lam.create_event_source_mapping(FunctionName=fn, EventSourceArn=arn,
StartingPosition="TRIM_HORIZON")["UUID"]
received, deadline = [], time.time() + 30
while time.time() < deadline and len(received) < 3:
try:
events = logs.filter_log_events(logGroupName=f"/aws/lambda/{fn}")["events"]
except ClientError:
events = []
received = [line[len("XREC "):] for e in events for line in e["message"].splitlines()
if line.startswith("XREC ")]
time.sleep(0.5)
assert received == ["rec-0", "rec-1", "rec-2"]
lam.delete_event_source_mapping(UUID=esm)
finally:
lam.delete_function(FunctionName=fn)
owner.delete_stream(StreamName=stream)
def test_kinesis_mapping_read_needs_every_read_in_the_stream_policy():
"""The stream policy must allow the function's role session every read a mapping makes."""
from ministack.core.responses import request_scope
from ministack.services import kinesis as kmod
owner, consumer, name = "111111111111", "222222222222", f"pol-{_uuid_mod.uuid4().hex[:8]}"
arn = f"arn:aws:kinesis:us-east-1:{owner}:stream/{name}"
role = f"arn:aws:iam::{consumer}:role/reader"
with request_scope(owner, "us-east-1"):
kmod._create_stream({"StreamName": name, "ShardCount": 1})
try:
assert kmod.mapping_read_denied(arn, role, "fn", consumer)[1] == "kinesis:DescribeStream"
reads = ["kinesis:DescribeStream", "kinesis:DescribeStreamSummary", "kinesis:GetRecords",
"kinesis:GetShardIterator", "kinesis:ListShards"]
def grant(actions):
with request_scope(owner, "us-east-1"):
kmod._resource_policies[arn] = json.dumps({"Version": "2012-10-17", "Statement": [{
"Effect": "Allow", "Principal": {"AWS": role}, "Action": actions, "Resource": arn}]})
grant(reads[:-1])
assert kmod.mapping_read_denied(arn, role, "fn", consumer)[1] == "kinesis:ListShards"
grant(reads)
assert kmod.mapping_read_denied(arn, role, "fn", consumer) is None
assert kmod.mapping_read_denied(arn, f"arn:aws:iam::{consumer}:role/other", "fn", consumer) is not None
finally:
with request_scope(owner, "us-east-1"):
kmod._resource_policies.pop(arn, None)
kmod._streams.pop(name, None)