Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
Add AWS::EventBridge::Pipes provisioner (#354)
David Teasdale336dfdc6f721
diff --git a/ministack/app.py b/ministack/app.pyindex 52bc2947..4cd4ab3e 100644--- a/ministack/app.py+++ b/ministack/app.py@@ -884,6 +884,7 @@ def _reset_all_state():"emr", "alb", "acm", "ses_v2", "waf", "efs", "cloudformation", "kms","cloudfront", "codebuild", "ecr", "appsync", "servicediscovery","rds_data", "s3files", "appconfig", "transfer", "scheduler", "autoscaling", "eks", "iam",+ "pipes"]for mod_name in _ALL_SERVICE_MODULES:if mod_name in _loaded_modules:diff --git a/ministack/services/cloudformation/provisioners.py b/ministack/services/cloudformation/provisioners.pyindex 08552f16..cd79d73f 100644--- a/ministack/services/cloudformation/provisioners.py+++ b/ministack/services/cloudformation/provisioners.py@@ -34,6 +34,7 @@ import ministack.services.ec2 as _ec2import ministack.services.ecs as _ecsimport ministack.services.alb as _albimport ministack.services.kinesis as _kinesis+import ministack.services.pipes as _pipesimport ministack.services.stepfunctions as _sfnimport ministack.services.route53 as _r53import ministack.services.apigateway as _apigw_v2@@ -244,7 +245,7 @@ def _sns_create(logical_id, props, stack_name):"endpoint": endpoint,"confirmed": protocol not in ("http", "https"),"owner": get_account_id(),- "attributes": {},+ "attributes": {}}_sns._topics[arn]["subscriptions"].append(sub)_sns._sub_arn_to_topic[sub_arn] = arn@@ -278,7 +279,14 @@ def _sns_sub_create(logical_id, props, stack_name):"endpoint": endpoint,"confirmed": protocol not in ("http", "https"),"owner": get_account_id(),- "attributes": {},+ "attributes": {+ "FilterPolicyScope": props.get("FilterPolicyScope", "MessageAttributes"),+ "FilterPolicy": (+ json.dumps(props.get("FilterPolicy"))+ if isinstance(props.get("FilterPolicy"), (dict, list))+ else (props.get("FilterPolicy", "") or "")+ ),+ },}topic["subscriptions"].append(sub)_sns._sub_arn_to_topic[sub_arn] = topic_arn@@ -1079,6 +1087,34 @@ def _lambda_esm_delete(physical_id, props):_lambda_svc._esms.pop(physical_id, None)+# --- EventBridge Pipes (minimal: DynamoDB Streams -> SNS) ---++def _pipes_pipe_create(logical_id, props, stack_name):+ name = props.get("Name") or _physical_name(stack_name, logical_id, max_len=64)+ source = props.get("Source", "")+ target = props.get("Target", "")+ role_arn = props.get("RoleArn", "")+ desired_state = props.get("DesiredState", "RUNNING")++ source_params = props.get("SourceParameters", {})+ ddb_params = source_params.get("DynamoDBStreamParameters", {}) if isinstance(source_params, dict) else {}+ starting_position = ddb_params.get("StartingPosition", "LATEST")++ pipe = _pipes.register_pipe(+ name=name,+ source=source,+ target=target,+ role_arn=role_arn,+ desired_state=desired_state,+ starting_position=starting_position,+ )+ return name, {"Arn": pipe["Arn"], "Name": name}+++def _pipes_pipe_delete(physical_id, props):+ _pipes.delete_pipe(physical_id)++# --- Lambda Alias ---def _lambda_alias_create(logical_id, props, stack_name):@@ -2786,6 +2822,7 @@ _RESOURCE_HANDLERS = {"AWS::ApiGateway::Deployment": {"create": _apigw_deployment_create, "delete": _apigw_deployment_delete},"AWS::ApiGateway::Stage": {"create": _apigw_stage_create, "delete": _apigw_stage_delete},"AWS::Lambda::EventSourceMapping": {"create": _lambda_esm_create, "delete": _lambda_esm_delete},+ "AWS::Pipes::Pipe": {"create": _pipes_pipe_create, "delete": _pipes_pipe_delete},"AWS::Lambda::Alias": {"create": _lambda_alias_create, "delete": _lambda_alias_delete},"AWS::SQS::QueuePolicy": {"create": _sqs_queue_policy_create, "delete": _sqs_queue_policy_delete},"AWS::SNS::TopicPolicy": {"create": _sns_topic_policy_create, "delete": _sns_topic_policy_delete},diff --git a/ministack/services/pipes.py b/ministack/services/pipes.pynew file mode 100644index 00000000..3e44ea5c--- /dev/null+++ b/ministack/services/pipes.py@@ -0,0 +1,173 @@+"""+Minimal EventBridge Pipes runtime for local CloudFormation use.++Scope intentionally limited to:+- Source: DynamoDB Streams+- Target: SNS+- Lifecycle: create/delete (via CloudFormation handlers)+"""++import copy+import json+import logging+import os+import threading+import time++from ministack.core.responses import AccountScopedDict, get_account_id, new_uuid++logger = logging.getLogger("pipes")++REGION = os.environ.get("MINISTACK_REGION", "us-east-1")++_pipes = AccountScopedDict() # pipe_name -> pipe record+_positions = AccountScopedDict() # pipe_arn -> next stream record index+_poller_started = False+_poller_lock = threading.Lock()+++def get_state():+ return {+ "pipes": copy.deepcopy(_pipes),+ "positions": copy.deepcopy(_positions),+ }+++def reset():+ _pipes.clear()+ _positions.clear()+++def register_pipe(+ *,+ name: str,+ source: str,+ target: str,+ role_arn: str = "",+ desired_state: str = "RUNNING",+ starting_position: str = "LATEST",+ tags: dict | None = None,+):+ arn = f"arn:aws:pipes:{REGION}:{get_account_id()}:pipe/{name}"+ state = "STOPPED" if str(desired_state).upper() == "STOPPED" else "RUNNING"+ start = str(starting_position or "LATEST").upper()+ if start not in ("LATEST", "TRIM_HORIZON"):+ start = "LATEST"++ _pipes[name] = {+ "Name": name,+ "Arn": arn,+ "RoleArn": role_arn,+ "Source": source,+ "Target": target,+ "DesiredState": state,+ "CurrentState": state,+ "StartingPosition": start,+ "Tags": tags or {},+ "CreationTime": time.time(),+ }+ _positions[arn] = _initial_position(_pipes[name])++ _ensure_poller()+ return _pipes[name]+++def delete_pipe(name: str):+ pipe = _pipes.pop(name, None)+ if pipe:+ _positions.pop(pipe["Arn"], None)+++def _ensure_poller():+ global _poller_started+ with _poller_lock:+ if not _poller_started:+ t = threading.Thread(target=_poll_loop, daemon=True)+ t.start()+ _poller_started = True+++def _poll_loop():+ while True:+ try:+ _poll_once()+ except Exception as e:+ logger.error("Pipes poller error: %s", e)+ time.sleep(1 if _pipes else 5)+++def _poll_once():+ from ministack.services import dynamodb as _ddb++ stream_records = getattr(_ddb, "_stream_records", None)+ if stream_records is None:+ return++ for pipe in list(_pipes.values()):+ if pipe.get("CurrentState") != "RUNNING":+ continue++ source_arn = pipe.get("Source", "")+ target_arn = pipe.get("Target", "")+ if ":dynamodb:" not in source_arn or "/stream/" not in source_arn:+ continue+ if ":sns:" not in target_arn:+ continue++ table_name = _table_name_from_stream_arn(source_arn)+ if not table_name:+ continue++ records = stream_records.get(table_name, [])+ pos = int(_positions.get(pipe["Arn"], 0))+ if pos < 0:+ pos = 0+ if pos >= len(records):+ continue++ batch = records[pos:]+ for rec in batch:+ _publish_record_to_sns(target_arn, pipe, rec)+ _positions[pipe["Arn"]] = pos + len(batch)+++def _publish_record_to_sns(topic_arn: str, pipe: dict, record: dict):+ from ministack.services import sns as _sns++ topic = _sns._topics.get(topic_arn)+ if not topic:+ logger.warning("Pipes %s: SNS topic not found %s", pipe.get("Name"), topic_arn)+ return++ msg_id = new_uuid()+ message = json.dumps(record)+ subject = f"Pipes {pipe.get('Name', '')}"++ topic["messages"].append({+ "id": msg_id,+ "message": message,+ "subject": subject,+ "message_structure": "",+ "message_attributes": {},+ "timestamp": int(time.time()),+ })+ _sns._fanout(topic_arn, msg_id, message, subject, "", {})+++def _table_name_from_stream_arn(stream_arn: str) -> str:+ if "/stream/" not in stream_arn:+ return ""+ return stream_arn.split("/stream/", 1)[0].rsplit("/", 1)[-1]+++def _initial_position(pipe: dict) -> int:+ from ministack.services import dynamodb as _ddb++ table_name = _table_name_from_stream_arn(pipe.get("Source", ""))+ if not table_name:+ return 0++ records = getattr(_ddb, "_stream_records", {}).get(table_name, [])+ if pipe.get("StartingPosition") == "TRIM_HORIZON":+ return 0+ return len(records)diff --git a/tests/test_cfn.py b/tests/test_cfn.pyindex 7df8128a..f93f80a2 100644--- a/tests/test_cfn.py+++ b/tests/test_cfn.py@@ -1392,6 +1392,158 @@ def test_cfn_dynamodb_stream_spec(cfn, ddb):_wait_stack(cfn, stack_name)+def test_cfn_pipes_dynamodb_stream_to_sns(cfn, ddb, sqs):+ uid = _uuid_mod.uuid4().hex[:8]+ stack_name = f"cfn-pipe-{uid}"+ table_name = f"cfn-pipe-table-{uid}"+ queue_name = f"cfn-pipe-q-{uid}"+ topic_name = f"cfn-pipe-topic-{uid}"++ template = {+ "AWSTemplateFormatVersion": "2010-09-09",+ "Resources": {+ "PipeTable": {+ "Type": "AWS::DynamoDB::Table",+ "Properties": {+ "TableName": table_name,+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],+ "BillingMode": "PAY_PER_REQUEST",+ "StreamSpecification": {"StreamViewType": "NEW_AND_OLD_IMAGES"},+ },+ },+ "PipeTopic": {+ "Type": "AWS::SNS::Topic",+ "Properties": {"TopicName": topic_name},+ },+ "PipeQueue": {+ "Type": "AWS::SQS::Queue",+ "Properties": {"QueueName": queue_name},+ },+ "PipeSubscription": {+ "Type": "AWS::SNS::Subscription",+ "Properties": {+ "Protocol": "sqs",+ "TopicArn": {"Ref": "PipeTopic"},+ "Endpoint": {"Fn::GetAtt": ["PipeQueue", "Arn"]},+ },+ },+ "DdbToSnsPipe": {+ "Type": "AWS::Pipes::Pipe",+ "Properties": {+ "Name": f"{stack_name}-pipe",+ "RoleArn": "arn:aws:iam::000000000000:role/test-pipe-role",+ "Source": {"Fn::GetAtt": ["PipeTable", "StreamArn"]},+ "Target": {"Ref": "PipeTopic"},+ "SourceParameters": {+ "DynamoDBStreamParameters": {"StartingPosition": "TRIM_HORIZON"}+ },+ },+ },+ },+ }++ cfn.create_stack(StackName=stack_name, TemplateBody=json.dumps(template))+ stack = _wait_stack(cfn, stack_name)+ assert stack["StackStatus"] == "CREATE_COMPLETE", stack.get("StackStatusReason")++ queue_url = sqs.get_queue_url(QueueName=queue_name)["QueueUrl"]++ ddb.put_item(+ TableName=table_name,+ Item={+ "pk": {"S": "1"},+ "val": {"S": "hello"},+ },+ )++ msg = None+ deadline = time.time() + 8+ while time.time() < deadline:+ out = sqs.receive_message(QueueUrl=queue_url, MaxNumberOfMessages=1, WaitTimeSeconds=1)+ msgs = out.get("Messages", [])+ if msgs:+ msg = msgs[0]+ break++ assert msg is not None, "Expected DynamoDB stream record to reach SNS/SQS via Pipe"++ envelope = json.loads(msg["Body"])+ rec = json.loads(envelope["Message"])+ assert rec.get("eventSource") == "aws:dynamodb"+ assert rec.get("eventName") in ("INSERT", "MODIFY", "REMOVE")++ dynamodb = rec.get("dynamodb", {})+ assert dynamodb.get("Keys", {}).get("pk", {}).get("S") == "1"+ assert dynamodb.get("NewImage", {}).get("pk", {}).get("S") == "1"++ cfn.delete_stack(StackName=stack_name)+ _wait_stack(cfn, stack_name)+++def test_cfn_sns_topic_subscription_filter_policy_scope(cfn, sns, sqs):+ uid = _uuid_mod.uuid4().hex[:8]+ stack_name = f"cfn-sns-filter-{uid}"+ queue_name = f"cfn-sns-filter-q-{uid}"+ topic_name = f"cfn-sns-filter-topic-{uid}"++ template = {+ "AWSTemplateFormatVersion": "2010-09-09",+ "Resources": {+ "FilterQueue": {+ "Type": "AWS::SQS::Queue",+ "Properties": {"QueueName": queue_name},+ },+ "FilterTopic": {+ "Type": "AWS::SNS::Topic",+ "Properties": {+ "TopicName": topic_name,+ },+ },+ "FilterSubscription": {+ "Type": "AWS::SNS::Subscription",+ "Properties": {+ "Protocol": "sqs",+ "TopicArn": {"Ref": "FilterTopic"},+ "Endpoint": {"Fn::GetAtt": ["FilterQueue", "Arn"]},+ "FilterPolicy": {"color": ["blue"]},+ },+ },+ },+ "Outputs": {+ "TopicArn": {"Value": {"Ref": "FilterTopic"}},+ },+ }++ cfn.create_stack(StackName=stack_name, TemplateBody=json.dumps(template))+ stack = _wait_stack(cfn, stack_name)+ assert stack["StackStatus"] == "CREATE_COMPLETE", stack.get("StackStatusReason")++ outputs = {o["OutputKey"]: o["OutputValue"] for o in stack.get("Outputs", [])}+ topic_arn = outputs["TopicArn"]+ queue_url = sqs.get_queue_url(QueueName=queue_name)["QueueUrl"]++ sns.publish(+ TopicArn=topic_arn,+ Message="red message",+ MessageAttributes={"color": {"DataType": "String", "StringValue": "red"}},+ )+ msgs = sqs.receive_message(QueueUrl=queue_url, MaxNumberOfMessages=1, WaitTimeSeconds=0)+ assert len(msgs.get("Messages", [])) == 0++ sns.publish(+ TopicArn=topic_arn,+ Message="blue message",+ MessageAttributes={"color": {"DataType": "String", "StringValue": "blue"}},+ )+ msgs = sqs.receive_message(QueueUrl=queue_url, MaxNumberOfMessages=1, WaitTimeSeconds=1)+ assert len(msgs.get("Messages", [])) == 1+ body = json.loads(msgs["Messages"][0]["Body"])+ assert body["Message"] == "blue message"++ cfn.delete_stack(StackName=stack_name)+ _wait_stack(cfn, stack_name)+# ===========================================================================# CodeBuild Project Tests# ===========================================================================