Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
Regionalize Pipes state (#1062)
Ian Oberst3d0e9b08e129
diff --git a/ministack/services/pipes.py b/ministack/services/pipes.pyindex a82d2224..fd6efc8a 100644--- a/ministack/services/pipes.py+++ b/ministack/services/pipes.py@@ -16,14 +16,22 @@ import timefrom ministack.core.arn import ArnParseError, parse_arnfrom ministack.core.persistence import load_state-from ministack.core.responses import AccountScopedDict, get_account_id, get_region, new_uuid+from ministack.core.responses import (+ AccountRegionScopedDict,+ AccountScopedDict,+ _request_account_id,+ _request_region,+ get_account_id,+ get_region,+ new_uuid,+)logger = logging.getLogger("pipes")REGION = os.environ.get("MINISTACK_REGION", "us-east-1")-_pipes = AccountScopedDict() # pipe_name -> pipe record-_positions = AccountScopedDict() # pipe_arn -> next stream record index+_pipes = AccountRegionScopedDict() # pipe_name -> pipe record+_positions = AccountRegionScopedDict() # pipe_arn -> next stream record index_poller_started = False_poller_lock = threading.Lock()@@ -37,16 +45,80 @@ def get_state():def restore_state(data):if data:- _pipes.update(data.get("pipes", {}))- _positions.update(data.get("positions", {}))+ _restore_pipe_store(data.get("pipes", {}))+ _restore_position_store(data.get("positions", {}))# Restored RUNNING pipes need the background poller — register_pipe# is the only other place that starts it, and it isn't called on# warm-boot. Without this, persisted pipes would silently stop# forwarding events until a new pipe is registered.- if any(p.get("CurrentState") == "RUNNING" for p in _pipes.values()):+ if any(p.get("CurrentState") == "RUNNING" for p in _pipes.all_values()):_ensure_poller()+def _pipe_arn_scope(pipe_arn: str, default_account_id: str | None = None) -> tuple[str, str]:+ try:+ spec = parse_arn(pipe_arn)+ except ArnParseError:+ return default_account_id or get_account_id(), get_region()+ if spec.service != "pipes":+ return default_account_id or get_account_id(), get_region()+ return spec.account_id or default_account_id or get_account_id(), spec.region or get_region()+++def _pipe_record_scope(pipe: dict, default_account_id: str | None = None) -> tuple[str, str]:+ return _pipe_arn_scope(pipe.get("Arn", ""), default_account_id)+++def _restore_pipe_store(data) -> None:+ if isinstance(data, AccountRegionScopedDict):+ _pipes.update(data)+ return+ if isinstance(data, AccountScopedDict):+ for (account_id, name), pipe in data._data.items():+ restored_account_id, region = _pipe_record_scope(pipe, account_id)+ _pipes.set_scoped(restored_account_id, region, name, copy.deepcopy(pipe))+ return+ if isinstance(data, dict):+ for key, pipe in data.items():+ if isinstance(key, tuple) and len(key) == 3:+ account_id, region, name = key+ elif isinstance(key, tuple) and len(key) == 2:+ account_id, name = key+ account_id, region = _pipe_record_scope(pipe, account_id)+ else:+ name = key+ account_id, region = _pipe_record_scope(pipe)+ _pipes.set_scoped(account_id, region, name, copy.deepcopy(pipe))+++def _restore_position_store(data) -> None:+ if isinstance(data, AccountRegionScopedDict):+ _positions.update(data)+ return+ if isinstance(data, AccountScopedDict):+ for (account_id, pipe_arn), position in data._data.items():+ restored_account_id, region = _pipe_arn_scope(pipe_arn, account_id)+ _positions.set_scoped(restored_account_id, region, pipe_arn, position)+ return+ if isinstance(data, dict):+ for key, position in data.items():+ if isinstance(key, tuple) and len(key) == 3:+ account_id, region, pipe_arn = key+ elif isinstance(key, tuple) and len(key) == 2:+ account_id, pipe_arn = key+ account_id, region = _pipe_arn_scope(pipe_arn, account_id)+ else:+ pipe_arn = key+ account_id, region = _pipe_arn_scope(pipe_arn)+ _positions.set_scoped(account_id, region, pipe_arn, position)+++def _iter_all_pipes():+ for scoped_key, pipe in list(_pipes.all_items()):+ account_id, region, _name = scoped_key+ yield account_id, region, pipe++try:_restored = load_state("pipes")if _restored:@@ -115,7 +187,7 @@ def _poll_loop():_poll_once()except Exception as e:logger.error("Pipes poller error: %s", e)- time.sleep(1 if _pipes else 5)+ time.sleep(1 if _pipes.has_any() else 5)def _poll_once():@@ -125,36 +197,41 @@ def _poll_once():if stream_records is None:return- for pipe in list(_pipes.values()):- if pipe.get("CurrentState") != "RUNNING":- continue-- source_arn = pipe.get("Source", "")- target_arn = pipe.get("Target", "")- if _arn_service(source_arn) != "dynamodb" or _arn_service(target_arn) != "sns":- continue-- source = _dynamodb_stream_source(source_arn)- if source is None:- continue- source_spec, table_name = source- pipe_account_id = _pipe_account_id(pipe)- if source_spec.account_id != pipe_account_id:- continue-- records = stream_records.get_scoped(- pipe_account_id, source_spec.region, table_name, []- )- pos = int(_positions.get(pipe["Arn"], 0))- if pos < 0:- pos = 0- if pos >= len(records):- continue-- batch = records[pos:]- for rec in batch:- _publish_record_to_sns(target_arn, pipe, rec)- _positions[pipe["Arn"]] = pos + len(batch)+ for pipe_account_id, pipe_region, pipe in _iter_all_pipes():+ account_token = _request_account_id.set(pipe_account_id)+ region_token = _request_region.set(pipe_region)+ try:+ if pipe.get("CurrentState") != "RUNNING":+ continue++ source_arn = pipe.get("Source", "")+ target_arn = pipe.get("Target", "")+ if _arn_service(source_arn) != "dynamodb" or _arn_service(target_arn) != "sns":+ continue++ source = _dynamodb_stream_source(source_arn)+ if source is None:+ continue+ source_spec, table_name = source+ if source_spec.account_id != pipe_account_id:+ continue++ records = stream_records.get_scoped(+ pipe_account_id, source_spec.region, table_name, []+ )+ pos = int(_positions.get(pipe["Arn"], 0))+ if pos < 0:+ pos = 0+ if pos >= len(records):+ continue++ batch = records[pos:]+ for rec in batch:+ _publish_record_to_sns(target_arn, pipe, rec)+ _positions[pipe["Arn"]] = pos + len(batch)+ finally:+ _request_region.reset(region_token)+ _request_account_id.reset(account_token)def _publish_record_to_sns(topic_arn: str, pipe: dict, record: dict):diff --git a/tests/test_pipes.py b/tests/test_pipes.pyindex f60e0e66..fda138c4 100644--- a/tests/test_pipes.py+++ b/tests/test_pipes.py@@ -113,3 +113,139 @@ def test_pipes_dynamodb_stream_read_rejects_cross_account_source(monkeypatch):finally:_pipes.reset()_ddb._stream_records.clear()+++def test_pipes_poller_processes_same_name_pipes_in_each_region(monkeypatch):+ from ministack.core.responses import (+ get_account_id,+ get_region,+ set_request_account_id,+ set_request_region,+ )++ original_account = get_account_id()+ original_region = get_region()+ account_id = "000000000000"+ table_name = "SamePipeTable"++ _pipes.reset()+ _ddb._stream_records.clear()+ try:+ set_request_account_id(account_id)+ for region, label in (("us-east-1", "east"), ("us-west-2", "west")):+ source_arn = (+ f"arn:aws:dynamodb:{region}:{account_id}:"+ f"table/{table_name}/stream/2026-05-22T00:00:00.000"+ )+ target_arn = f"arn:aws:sns:{region}:{account_id}:PipeTopic"+ pipe_arn = f"arn:aws:pipes:{region}:{account_id}:pipe/SamePipe"+ record = {+ "eventID": f"evt-{label}",+ "eventSource": "aws:dynamodb",+ }++ _ddb._stream_records.set_scoped(account_id, region, table_name, [record])+ _pipes._pipes.set_scoped(+ account_id,+ region,+ "SamePipe",+ {+ "Name": "SamePipe",+ "Arn": pipe_arn,+ "Source": source_arn,+ "Target": target_arn,+ "CurrentState": "RUNNING",+ "StartingPosition": "TRIM_HORIZON",+ },+ )+ _pipes._positions.set_scoped(account_id, region, pipe_arn, 0)++ set_request_region("eu-central-1")+ delivered = []+ monkeypatch.setattr(+ _pipes,+ "_publish_record_to_sns",+ lambda topic_arn, pipe, record: delivered.append(+ (topic_arn, pipe["Arn"], record["eventID"])+ ),+ )++ _pipes._poll_once()++ assert sorted(delivered) == [+ (+ f"arn:aws:sns:us-east-1:{account_id}:PipeTopic",+ f"arn:aws:pipes:us-east-1:{account_id}:pipe/SamePipe",+ "evt-east",+ ),+ (+ f"arn:aws:sns:us-west-2:{account_id}:PipeTopic",+ f"arn:aws:pipes:us-west-2:{account_id}:pipe/SamePipe",+ "evt-west",+ ),+ ]+ assert _pipes._positions.get_scoped(+ account_id,+ "us-east-1",+ f"arn:aws:pipes:us-east-1:{account_id}:pipe/SamePipe",+ ) == 1+ assert _pipes._positions.get_scoped(+ account_id,+ "us-west-2",+ f"arn:aws:pipes:us-west-2:{account_id}:pipe/SamePipe",+ ) == 1+ finally:+ _pipes.reset()+ _ddb._stream_records.clear()+ set_request_account_id(original_account)+ set_request_region(original_region)+++def test_pipes_restore_legacy_account_scoped_state_uses_pipe_arn_region():+ from ministack.core.responses import (+ AccountScopedDict,+ get_account_id,+ get_region,+ set_request_account_id,+ set_request_region,+ )++ original_account = get_account_id()+ original_region = get_region()+ account_id = "000000000000"+ pipe_arn = f"arn:aws:pipes:us-west-2:{account_id}:pipe/LegacyPipe"++ legacy_pipes = AccountScopedDict()+ legacy_pipes._data[(account_id, "LegacyPipe")] = {+ "Name": "LegacyPipe",+ "Arn": pipe_arn,+ "Source": (+ f"arn:aws:dynamodb:us-west-2:{account_id}:"+ "table/LegacyPipeTable/stream/2026-05-22T00:00:00.000"+ ),+ "Target": f"arn:aws:sns:us-west-2:{account_id}:PipeTopic",+ "CurrentState": "STOPPED",+ "StartingPosition": "LATEST",+ }+ legacy_positions = AccountScopedDict()+ legacy_positions._data[(account_id, pipe_arn)] = 7++ _pipes.reset()+ try:+ set_request_account_id(account_id)+ set_request_region("us-east-1")++ _pipes.restore_state({+ "pipes": legacy_pipes,+ "positions": legacy_positions,+ })++ assert _pipes._pipes.get_scoped(account_id, "us-east-1", "LegacyPipe") is None+ assert _pipes._pipes.get_scoped(account_id, "us-west-2", "LegacyPipe")[+ "Arn"+ ] == pipe_arn+ assert _pipes._positions.get_scoped(account_id, "us-west-2", pipe_arn) == 7+ finally:+ _pipes.reset()+ set_request_account_id(original_account)+ set_request_region(original_region)