Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

Regionalize Pipes state (#1062)

Ian Oberst3d0e9b08e129
diff --git a/ministack/services/pipes.py b/ministack/services/pipes.py
index a82d2224..fd6efc8a 100644
--- a/ministack/services/pipes.py
+++ b/ministack/services/pipes.py
@@ -16,14 +16,22 @@ import time
from ministack.core.arn import ArnParseError, parse_arn
from ministack.core.persistence import load_state
-from ministack.core.responses import AccountScopedDict, get_account_id, get_region, new_uuid
+from ministack.core.responses import (
+ AccountRegionScopedDict,
+ AccountScopedDict,
+ _request_account_id,
+ _request_region,
+ get_account_id,
+ get_region,
+ new_uuid,
+)
logger = logging.getLogger("pipes")
REGION = os.environ.get("MINISTACK_REGION", "us-east-1")
-_pipes = AccountScopedDict() # pipe_name -> pipe record
-_positions = AccountScopedDict() # pipe_arn -> next stream record index
+_pipes = AccountRegionScopedDict() # pipe_name -> pipe record
+_positions = AccountRegionScopedDict() # pipe_arn -> next stream record index
_poller_started = False
_poller_lock = threading.Lock()
@@ -37,16 +45,80 @@ def get_state():
def restore_state(data):
if data:
- _pipes.update(data.get("pipes", {}))
- _positions.update(data.get("positions", {}))
+ _restore_pipe_store(data.get("pipes", {}))
+ _restore_position_store(data.get("positions", {}))
# Restored RUNNING pipes need the background poller — register_pipe
# is the only other place that starts it, and it isn't called on
# warm-boot. Without this, persisted pipes would silently stop
# forwarding events until a new pipe is registered.
- if any(p.get("CurrentState") == "RUNNING" for p in _pipes.values()):
+ if any(p.get("CurrentState") == "RUNNING" for p in _pipes.all_values()):
_ensure_poller()
+def _pipe_arn_scope(pipe_arn: str, default_account_id: str | None = None) -> tuple[str, str]:
+ try:
+ spec = parse_arn(pipe_arn)
+ except ArnParseError:
+ return default_account_id or get_account_id(), get_region()
+ if spec.service != "pipes":
+ return default_account_id or get_account_id(), get_region()
+ return spec.account_id or default_account_id or get_account_id(), spec.region or get_region()
+
+
+def _pipe_record_scope(pipe: dict, default_account_id: str | None = None) -> tuple[str, str]:
+ return _pipe_arn_scope(pipe.get("Arn", ""), default_account_id)
+
+
+def _restore_pipe_store(data) -> None:
+ if isinstance(data, AccountRegionScopedDict):
+ _pipes.update(data)
+ return
+ if isinstance(data, AccountScopedDict):
+ for (account_id, name), pipe in data._data.items():
+ restored_account_id, region = _pipe_record_scope(pipe, account_id)
+ _pipes.set_scoped(restored_account_id, region, name, copy.deepcopy(pipe))
+ return
+ if isinstance(data, dict):
+ for key, pipe in data.items():
+ if isinstance(key, tuple) and len(key) == 3:
+ account_id, region, name = key
+ elif isinstance(key, tuple) and len(key) == 2:
+ account_id, name = key
+ account_id, region = _pipe_record_scope(pipe, account_id)
+ else:
+ name = key
+ account_id, region = _pipe_record_scope(pipe)
+ _pipes.set_scoped(account_id, region, name, copy.deepcopy(pipe))
+
+
+def _restore_position_store(data) -> None:
+ if isinstance(data, AccountRegionScopedDict):
+ _positions.update(data)
+ return
+ if isinstance(data, AccountScopedDict):
+ for (account_id, pipe_arn), position in data._data.items():
+ restored_account_id, region = _pipe_arn_scope(pipe_arn, account_id)
+ _positions.set_scoped(restored_account_id, region, pipe_arn, position)
+ return
+ if isinstance(data, dict):
+ for key, position in data.items():
+ if isinstance(key, tuple) and len(key) == 3:
+ account_id, region, pipe_arn = key
+ elif isinstance(key, tuple) and len(key) == 2:
+ account_id, pipe_arn = key
+ account_id, region = _pipe_arn_scope(pipe_arn, account_id)
+ else:
+ pipe_arn = key
+ account_id, region = _pipe_arn_scope(pipe_arn)
+ _positions.set_scoped(account_id, region, pipe_arn, position)
+
+
+def _iter_all_pipes():
+ for scoped_key, pipe in list(_pipes.all_items()):
+ account_id, region, _name = scoped_key
+ yield account_id, region, pipe
+
+
try:
_restored = load_state("pipes")
if _restored:
@@ -115,7 +187,7 @@ def _poll_loop():
_poll_once()
except Exception as e:
logger.error("Pipes poller error: %s", e)
- time.sleep(1 if _pipes else 5)
+ time.sleep(1 if _pipes.has_any() else 5)
def _poll_once():
@@ -125,36 +197,41 @@ def _poll_once():
if stream_records is None:
return
- for pipe in list(_pipes.values()):
- if pipe.get("CurrentState") != "RUNNING":
- continue
-
- source_arn = pipe.get("Source", "")
- target_arn = pipe.get("Target", "")
- if _arn_service(source_arn) != "dynamodb" or _arn_service(target_arn) != "sns":
- continue
-
- source = _dynamodb_stream_source(source_arn)
- if source is None:
- continue
- source_spec, table_name = source
- pipe_account_id = _pipe_account_id(pipe)
- if source_spec.account_id != pipe_account_id:
- continue
-
- records = stream_records.get_scoped(
- pipe_account_id, source_spec.region, table_name, []
- )
- pos = int(_positions.get(pipe["Arn"], 0))
- if pos < 0:
- pos = 0
- if pos >= len(records):
- continue
-
- batch = records[pos:]
- for rec in batch:
- _publish_record_to_sns(target_arn, pipe, rec)
- _positions[pipe["Arn"]] = pos + len(batch)
+ for pipe_account_id, pipe_region, pipe in _iter_all_pipes():
+ account_token = _request_account_id.set(pipe_account_id)
+ region_token = _request_region.set(pipe_region)
+ try:
+ if pipe.get("CurrentState") != "RUNNING":
+ continue
+
+ source_arn = pipe.get("Source", "")
+ target_arn = pipe.get("Target", "")
+ if _arn_service(source_arn) != "dynamodb" or _arn_service(target_arn) != "sns":
+ continue
+
+ source = _dynamodb_stream_source(source_arn)
+ if source is None:
+ continue
+ source_spec, table_name = source
+ if source_spec.account_id != pipe_account_id:
+ continue
+
+ records = stream_records.get_scoped(
+ pipe_account_id, source_spec.region, table_name, []
+ )
+ pos = int(_positions.get(pipe["Arn"], 0))
+ if pos < 0:
+ pos = 0
+ if pos >= len(records):
+ continue
+
+ batch = records[pos:]
+ for rec in batch:
+ _publish_record_to_sns(target_arn, pipe, rec)
+ _positions[pipe["Arn"]] = pos + len(batch)
+ finally:
+ _request_region.reset(region_token)
+ _request_account_id.reset(account_token)
def _publish_record_to_sns(topic_arn: str, pipe: dict, record: dict):
diff --git a/tests/test_pipes.py b/tests/test_pipes.py
index f60e0e66..fda138c4 100644
--- a/tests/test_pipes.py
+++ b/tests/test_pipes.py
@@ -113,3 +113,139 @@ def test_pipes_dynamodb_stream_read_rejects_cross_account_source(monkeypatch):
finally:
_pipes.reset()
_ddb._stream_records.clear()
+
+
+def test_pipes_poller_processes_same_name_pipes_in_each_region(monkeypatch):
+ from ministack.core.responses import (
+ get_account_id,
+ get_region,
+ set_request_account_id,
+ set_request_region,
+ )
+
+ original_account = get_account_id()
+ original_region = get_region()
+ account_id = "000000000000"
+ table_name = "SamePipeTable"
+
+ _pipes.reset()
+ _ddb._stream_records.clear()
+ try:
+ set_request_account_id(account_id)
+ for region, label in (("us-east-1", "east"), ("us-west-2", "west")):
+ source_arn = (
+ f"arn:aws:dynamodb:{region}:{account_id}:"
+ f"table/{table_name}/stream/2026-05-22T00:00:00.000"
+ )
+ target_arn = f"arn:aws:sns:{region}:{account_id}:PipeTopic"
+ pipe_arn = f"arn:aws:pipes:{region}:{account_id}:pipe/SamePipe"
+ record = {
+ "eventID": f"evt-{label}",
+ "eventSource": "aws:dynamodb",
+ }
+
+ _ddb._stream_records.set_scoped(account_id, region, table_name, [record])
+ _pipes._pipes.set_scoped(
+ account_id,
+ region,
+ "SamePipe",
+ {
+ "Name": "SamePipe",
+ "Arn": pipe_arn,
+ "Source": source_arn,
+ "Target": target_arn,
+ "CurrentState": "RUNNING",
+ "StartingPosition": "TRIM_HORIZON",
+ },
+ )
+ _pipes._positions.set_scoped(account_id, region, pipe_arn, 0)
+
+ set_request_region("eu-central-1")
+ delivered = []
+ monkeypatch.setattr(
+ _pipes,
+ "_publish_record_to_sns",
+ lambda topic_arn, pipe, record: delivered.append(
+ (topic_arn, pipe["Arn"], record["eventID"])
+ ),
+ )
+
+ _pipes._poll_once()
+
+ assert sorted(delivered) == [
+ (
+ f"arn:aws:sns:us-east-1:{account_id}:PipeTopic",
+ f"arn:aws:pipes:us-east-1:{account_id}:pipe/SamePipe",
+ "evt-east",
+ ),
+ (
+ f"arn:aws:sns:us-west-2:{account_id}:PipeTopic",
+ f"arn:aws:pipes:us-west-2:{account_id}:pipe/SamePipe",
+ "evt-west",
+ ),
+ ]
+ assert _pipes._positions.get_scoped(
+ account_id,
+ "us-east-1",
+ f"arn:aws:pipes:us-east-1:{account_id}:pipe/SamePipe",
+ ) == 1
+ assert _pipes._positions.get_scoped(
+ account_id,
+ "us-west-2",
+ f"arn:aws:pipes:us-west-2:{account_id}:pipe/SamePipe",
+ ) == 1
+ finally:
+ _pipes.reset()
+ _ddb._stream_records.clear()
+ set_request_account_id(original_account)
+ set_request_region(original_region)
+
+
+def test_pipes_restore_legacy_account_scoped_state_uses_pipe_arn_region():
+ from ministack.core.responses import (
+ AccountScopedDict,
+ get_account_id,
+ get_region,
+ set_request_account_id,
+ set_request_region,
+ )
+
+ original_account = get_account_id()
+ original_region = get_region()
+ account_id = "000000000000"
+ pipe_arn = f"arn:aws:pipes:us-west-2:{account_id}:pipe/LegacyPipe"
+
+ legacy_pipes = AccountScopedDict()
+ legacy_pipes._data[(account_id, "LegacyPipe")] = {
+ "Name": "LegacyPipe",
+ "Arn": pipe_arn,
+ "Source": (
+ f"arn:aws:dynamodb:us-west-2:{account_id}:"
+ "table/LegacyPipeTable/stream/2026-05-22T00:00:00.000"
+ ),
+ "Target": f"arn:aws:sns:us-west-2:{account_id}:PipeTopic",
+ "CurrentState": "STOPPED",
+ "StartingPosition": "LATEST",
+ }
+ legacy_positions = AccountScopedDict()
+ legacy_positions._data[(account_id, pipe_arn)] = 7
+
+ _pipes.reset()
+ try:
+ set_request_account_id(account_id)
+ set_request_region("us-east-1")
+
+ _pipes.restore_state({
+ "pipes": legacy_pipes,
+ "positions": legacy_positions,
+ })
+
+ assert _pipes._pipes.get_scoped(account_id, "us-east-1", "LegacyPipe") is None
+ assert _pipes._pipes.get_scoped(account_id, "us-west-2", "LegacyPipe")[
+ "Arn"
+ ] == pipe_arn
+ assert _pipes._positions.get_scoped(account_id, "us-west-2", pipe_arn) == 7
+ finally:
+ _pipes.reset()
+ set_request_account_id(original_account)
+ set_request_region(original_region)