Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

Preserve Firehose region scope (#1100)

Ian Oberst13348f910cba
diff --git a/ministack/services/firehose.py b/ministack/services/firehose.py
index 0b1282cd..51223878 100644
--- a/ministack/services/firehose.py
+++ b/ministack/services/firehose.py
@@ -27,7 +27,7 @@ import time
from ministack.core.arn import ArnParseError, parse_arn
from ministack.core.persistence import PERSIST_STATE, load_state
from ministack.core.responses import (
- AccountScopedDict,
+ AccountRegionScopedDict,
error_response_json,
get_account_id,
get_region,
@@ -45,15 +45,15 @@ _S3_BUCKET_NAME_RE = re.compile(
# ─── in-memory state ──────────────────────────────────────────────────────────
-_streams = AccountScopedDict() # name -> stream descriptor
+_streams = AccountRegionScopedDict() # (account, region, name) -> stream descriptor
_lock = threading.Lock()
_dest_counter = 0
def reset():
- global _streams, _dest_counter
+ global _dest_counter
with _lock:
- _streams = {}
+ _streams.clear()
_dest_counter = 0
@@ -62,7 +62,7 @@ def get_state() -> dict:
def restore_state(data: dict):
- global _streams, _dest_counter
+ global _dest_counter
_streams.update(data.get("_streams", {}))
_dest_counter = data.get("_dest_counter", _dest_counter)
diff --git a/tests/test_firehose.py b/tests/test_firehose.py
index bddcfd94..ed55312b 100644
--- a/tests/test_firehose.py
+++ b/tests/test_firehose.py
@@ -6,9 +6,24 @@ import uuid as _uuid_mod
import zipfile
from urllib.parse import urlparse
+import boto3
import pytest
+from botocore.config import Config
from botocore.exceptions import ClientError
+ENDPOINT = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566")
+
+
+def _regional_client(service: str, region: str):
+ return boto3.client(
+ service,
+ endpoint_url=ENDPOINT,
+ aws_access_key_id="test",
+ aws_secret_access_key="test",
+ region_name=region,
+ config=Config(region_name=region, retries={"mode": "standard"}),
+ )
+
def test_firehose_create_and_describe(fh):
name = "intg-fh-basic"
@@ -675,3 +690,168 @@ def test_firehose_kinesis_source_with_lambda_processor(fh, s3, lam, kin):
except Exception: pass
try: kin.delete_stream(StreamName=stream)
except Exception: pass
+
+
+def test_firehose_same_name_streams_are_region_scoped(fh):
+ """Same-named delivery streams in two regions stay isolated with region-bearing ARNs."""
+ west = _regional_client("firehose", "us-west-2")
+ name = f"intg-fh-region-scope-{_uuid_mod.uuid4().hex[:8]}"
+
+ east_arn = fh.create_delivery_stream(
+ DeliveryStreamName=name, DeliveryStreamType="DirectPut"
+ )["DeliveryStreamARN"]
+ west_arn = west.create_delivery_stream(
+ DeliveryStreamName=name, DeliveryStreamType="DirectPut"
+ )["DeliveryStreamARN"]
+
+ assert east_arn == f"arn:aws:firehose:us-east-1:000000000000:deliverystream/{name}"
+ assert west_arn == f"arn:aws:firehose:us-west-2:000000000000:deliverystream/{name}"
+
+ east_desc = fh.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
+ west_desc = west.describe_delivery_stream(DeliveryStreamName=name)["DeliveryStreamDescription"]
+ assert east_desc["DeliveryStreamARN"] == east_arn
+ assert west_desc["DeliveryStreamARN"] == west_arn
+
+ # Use a high Limit so the assertion is robust to other streams the shared
+ # session fixture accumulates (default page size is 10).
+ assert name in fh.list_delivery_streams(Limit=10000)["DeliveryStreamNames"]
+ assert name in west.list_delivery_streams(Limit=10000)["DeliveryStreamNames"]
+
+
+def test_firehose_stream_names_do_not_fallback_across_region(fh):
+ """A stream created in one region is not visible from another region."""
+ west = _regional_client("firehose", "us-west-2")
+ name = f"intg-fh-no-fallback-{_uuid_mod.uuid4().hex[:8]}"
+
+ west.create_delivery_stream(DeliveryStreamName=name, DeliveryStreamType="DirectPut")
+
+ with pytest.raises(ClientError) as exc:
+ fh.describe_delivery_stream(DeliveryStreamName=name)
+ assert exc.value.response["Error"]["Code"] == "ResourceNotFoundException"
+
+ # West still resolves it, and it never leaks into the east listing.
+ assert name in west.list_delivery_streams(Limit=10000)["DeliveryStreamNames"]
+ assert name not in fh.list_delivery_streams(Limit=10000)["DeliveryStreamNames"]
+
+
+def test_firehose_restore_legacy_account_scoped_state_uses_arn_region():
+ """Legacy account-scoped persisted streams migrate to the region in their ARN."""
+ from ministack.core.responses import (
+ AccountScopedDict,
+ get_account_id,
+ get_region,
+ set_request_account_id,
+ set_request_region,
+ )
+ from ministack.services import firehose as _fh
+
+ original_account = get_account_id()
+ original_region = get_region()
+ account_id = "000000000000"
+ name = "LegacyFirehoseStream"
+ stream_arn = f"arn:aws:firehose:us-west-2:{account_id}:deliverystream/{name}"
+
+ legacy_streams = AccountScopedDict()
+ legacy_streams._data[(account_id, name)] = {
+ "name": name,
+ "arn": stream_arn,
+ "status": "ACTIVE",
+ "type": "DirectPut",
+ "version": 1,
+ "created_at": 1700000000,
+ "updated_at": 1700000000,
+ "destinations": [],
+ "tags": {},
+ "encryption": None,
+ "kinesis_source": None,
+ }
+
+ _fh.reset()
+ try:
+ set_request_account_id(account_id)
+ set_request_region("us-east-1")
+
+ _fh.restore_state({"_streams": legacy_streams})
+
+ # The legacy stream lands in the region carried by its ARN, not the
+ # active request region.
+ assert _fh._streams.get_scoped(account_id, "us-east-1", name) is None
+ assert _fh._streams.get_scoped(account_id, "us-west-2", name)["arn"] == stream_arn
+ finally:
+ _fh.reset()
+ set_request_account_id(original_account)
+ set_request_region(original_region)
+
+
+def test_firehose_kinesis_source_ingest_is_region_scoped():
+ """ingest_from_kinesis_source only fans out to same-region delivery streams.
+
+ A KinesisStreamAsSource delivery stream registered in us-west-2 must NOT be
+ matched by a Kinesis-source ingest running under a different request region,
+ even when the source ARN matches. Exercises the regional filtering inside
+ ``ingest_from_kinesis_source`` (``_streams.values()`` is scoped to the
+ current account+region), which the positive same-region tests don't cover.
+ """
+ from ministack.core.responses import (
+ get_account_id,
+ get_region,
+ set_request_account_id,
+ set_request_region,
+ )
+ from ministack.services import firehose as _fh
+
+ original_account = get_account_id()
+ original_region = get_region()
+ account_id = "000000000000"
+ name = "KinSrcRegionScoped"
+ source_arn = f"arn:aws:kinesis:us-west-2:{account_id}:stream/src"
+
+ def _make_stream():
+ return {
+ "name": name,
+ "arn": f"arn:aws:firehose:us-west-2:{account_id}:deliverystream/{name}",
+ "status": "ACTIVE",
+ "type": "KinesisStreamAsSource",
+ "version": 1,
+ "created_at": 1700000000,
+ "updated_at": 1700000000,
+ "destinations": [
+ {
+ "id": "destinationId-000000000001",
+ "type": "ExtendedS3",
+ "config": {"BucketARN": "arn:aws:s3:::no-such-bucket", "Prefix": "out/"},
+ "records": [],
+ }
+ ],
+ "tags": {},
+ "encryption": None,
+ "kinesis_source": {
+ "KinesisStreamARN": source_arn,
+ "RoleARN": "arn:aws:iam::000000000000:role/test",
+ "DeliveryStartTimestamp": 0,
+ },
+ }
+
+ _fh.reset()
+ try:
+ set_request_account_id(account_id)
+
+ # Register the delivery stream in us-west-2.
+ set_request_region("us-west-2")
+ west_stream = _make_stream()
+ _fh._streams.set_scoped(account_id, "us-west-2", name, west_stream)
+
+ # Ingest under a DIFFERENT region: the us-west-2 stream is out of scope,
+ # so nothing is delivered even though the source ARN matches.
+ set_request_region("us-east-1")
+ _fh.ingest_from_kinesis_source(source_arn, [("pk", b'{"a":1}')])
+ assert west_stream["destinations"][0]["records"] == []
+
+ # Control: same-region ingest matches and appends the record.
+ set_request_region("us-west-2")
+ _fh.ingest_from_kinesis_source(source_arn, [("pk", b'{"a":1}')])
+ assert len(west_stream["destinations"][0]["records"]) == 1
+ finally:
+ _fh.reset()
+ set_request_account_id(original_account)
+ set_request_region(original_region)