Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

Add AWS::EventBridge::Pipes provisioner (#354)

David Teasdale336dfdc6f721
diff --git a/ministack/app.py b/ministack/app.py
index 52bc2947..4cd4ab3e 100644
--- a/ministack/app.py
+++ b/ministack/app.py
@@ -884,6 +884,7 @@ def _reset_all_state():
"emr", "alb", "acm", "ses_v2", "waf", "efs", "cloudformation", "kms",
"cloudfront", "codebuild", "ecr", "appsync", "servicediscovery",
"rds_data", "s3files", "appconfig", "transfer", "scheduler", "autoscaling", "eks", "iam",
+ "pipes"
]
for mod_name in _ALL_SERVICE_MODULES:
if mod_name in _loaded_modules:
diff --git a/ministack/services/cloudformation/provisioners.py b/ministack/services/cloudformation/provisioners.py
index 08552f16..cd79d73f 100644
--- a/ministack/services/cloudformation/provisioners.py
+++ b/ministack/services/cloudformation/provisioners.py
@@ -34,6 +34,7 @@ import ministack.services.ec2 as _ec2
import ministack.services.ecs as _ecs
import ministack.services.alb as _alb
import ministack.services.kinesis as _kinesis
+import ministack.services.pipes as _pipes
import ministack.services.stepfunctions as _sfn
import ministack.services.route53 as _r53
import ministack.services.apigateway as _apigw_v2
@@ -244,7 +245,7 @@ def _sns_create(logical_id, props, stack_name):
"endpoint": endpoint,
"confirmed": protocol not in ("http", "https"),
"owner": get_account_id(),
- "attributes": {},
+ "attributes": {}
}
_sns._topics[arn]["subscriptions"].append(sub)
_sns._sub_arn_to_topic[sub_arn] = arn
@@ -278,7 +279,14 @@ def _sns_sub_create(logical_id, props, stack_name):
"endpoint": endpoint,
"confirmed": protocol not in ("http", "https"),
"owner": get_account_id(),
- "attributes": {},
+ "attributes": {
+ "FilterPolicyScope": props.get("FilterPolicyScope", "MessageAttributes"),
+ "FilterPolicy": (
+ json.dumps(props.get("FilterPolicy"))
+ if isinstance(props.get("FilterPolicy"), (dict, list))
+ else (props.get("FilterPolicy", "") or "")
+ ),
+ },
}
topic["subscriptions"].append(sub)
_sns._sub_arn_to_topic[sub_arn] = topic_arn
@@ -1079,6 +1087,34 @@ def _lambda_esm_delete(physical_id, props):
_lambda_svc._esms.pop(physical_id, None)
+# --- EventBridge Pipes (minimal: DynamoDB Streams -> SNS) ---
+
+def _pipes_pipe_create(logical_id, props, stack_name):
+ name = props.get("Name") or _physical_name(stack_name, logical_id, max_len=64)
+ source = props.get("Source", "")
+ target = props.get("Target", "")
+ role_arn = props.get("RoleArn", "")
+ desired_state = props.get("DesiredState", "RUNNING")
+
+ source_params = props.get("SourceParameters", {})
+ ddb_params = source_params.get("DynamoDBStreamParameters", {}) if isinstance(source_params, dict) else {}
+ starting_position = ddb_params.get("StartingPosition", "LATEST")
+
+ pipe = _pipes.register_pipe(
+ name=name,
+ source=source,
+ target=target,
+ role_arn=role_arn,
+ desired_state=desired_state,
+ starting_position=starting_position,
+ )
+ return name, {"Arn": pipe["Arn"], "Name": name}
+
+
+def _pipes_pipe_delete(physical_id, props):
+ _pipes.delete_pipe(physical_id)
+
+
# --- Lambda Alias ---
def _lambda_alias_create(logical_id, props, stack_name):
@@ -2786,6 +2822,7 @@ _RESOURCE_HANDLERS = {
"AWS::ApiGateway::Deployment": {"create": _apigw_deployment_create, "delete": _apigw_deployment_delete},
"AWS::ApiGateway::Stage": {"create": _apigw_stage_create, "delete": _apigw_stage_delete},
"AWS::Lambda::EventSourceMapping": {"create": _lambda_esm_create, "delete": _lambda_esm_delete},
+ "AWS::Pipes::Pipe": {"create": _pipes_pipe_create, "delete": _pipes_pipe_delete},
"AWS::Lambda::Alias": {"create": _lambda_alias_create, "delete": _lambda_alias_delete},
"AWS::SQS::QueuePolicy": {"create": _sqs_queue_policy_create, "delete": _sqs_queue_policy_delete},
"AWS::SNS::TopicPolicy": {"create": _sns_topic_policy_create, "delete": _sns_topic_policy_delete},
diff --git a/ministack/services/pipes.py b/ministack/services/pipes.py
new file mode 100644
index 00000000..3e44ea5c
--- /dev/null
+++ b/ministack/services/pipes.py
@@ -0,0 +1,173 @@
+"""
+Minimal EventBridge Pipes runtime for local CloudFormation use.
+
+Scope intentionally limited to:
+- Source: DynamoDB Streams
+- Target: SNS
+- Lifecycle: create/delete (via CloudFormation handlers)
+"""
+
+import copy
+import json
+import logging
+import os
+import threading
+import time
+
+from ministack.core.responses import AccountScopedDict, get_account_id, new_uuid
+
+logger = logging.getLogger("pipes")
+
+REGION = os.environ.get("MINISTACK_REGION", "us-east-1")
+
+_pipes = AccountScopedDict() # pipe_name -> pipe record
+_positions = AccountScopedDict() # pipe_arn -> next stream record index
+_poller_started = False
+_poller_lock = threading.Lock()
+
+
+def get_state():
+ return {
+ "pipes": copy.deepcopy(_pipes),
+ "positions": copy.deepcopy(_positions),
+ }
+
+
+def reset():
+ _pipes.clear()
+ _positions.clear()
+
+
+def register_pipe(
+ *,
+ name: str,
+ source: str,
+ target: str,
+ role_arn: str = "",
+ desired_state: str = "RUNNING",
+ starting_position: str = "LATEST",
+ tags: dict | None = None,
+):
+ arn = f"arn:aws:pipes:{REGION}:{get_account_id()}:pipe/{name}"
+ state = "STOPPED" if str(desired_state).upper() == "STOPPED" else "RUNNING"
+ start = str(starting_position or "LATEST").upper()
+ if start not in ("LATEST", "TRIM_HORIZON"):
+ start = "LATEST"
+
+ _pipes[name] = {
+ "Name": name,
+ "Arn": arn,
+ "RoleArn": role_arn,
+ "Source": source,
+ "Target": target,
+ "DesiredState": state,
+ "CurrentState": state,
+ "StartingPosition": start,
+ "Tags": tags or {},
+ "CreationTime": time.time(),
+ }
+ _positions[arn] = _initial_position(_pipes[name])
+
+ _ensure_poller()
+ return _pipes[name]
+
+
+def delete_pipe(name: str):
+ pipe = _pipes.pop(name, None)
+ if pipe:
+ _positions.pop(pipe["Arn"], None)
+
+
+def _ensure_poller():
+ global _poller_started
+ with _poller_lock:
+ if not _poller_started:
+ t = threading.Thread(target=_poll_loop, daemon=True)
+ t.start()
+ _poller_started = True
+
+
+def _poll_loop():
+ while True:
+ try:
+ _poll_once()
+ except Exception as e:
+ logger.error("Pipes poller error: %s", e)
+ time.sleep(1 if _pipes else 5)
+
+
+def _poll_once():
+ from ministack.services import dynamodb as _ddb
+
+ stream_records = getattr(_ddb, "_stream_records", None)
+ if stream_records is None:
+ return
+
+ for pipe in list(_pipes.values()):
+ if pipe.get("CurrentState") != "RUNNING":
+ continue
+
+ source_arn = pipe.get("Source", "")
+ target_arn = pipe.get("Target", "")
+ if ":dynamodb:" not in source_arn or "/stream/" not in source_arn:
+ continue
+ if ":sns:" not in target_arn:
+ continue
+
+ table_name = _table_name_from_stream_arn(source_arn)
+ if not table_name:
+ continue
+
+ records = stream_records.get(table_name, [])
+ pos = int(_positions.get(pipe["Arn"], 0))
+ if pos < 0:
+ pos = 0
+ if pos >= len(records):
+ continue
+
+ batch = records[pos:]
+ for rec in batch:
+ _publish_record_to_sns(target_arn, pipe, rec)
+ _positions[pipe["Arn"]] = pos + len(batch)
+
+
+def _publish_record_to_sns(topic_arn: str, pipe: dict, record: dict):
+ from ministack.services import sns as _sns
+
+ topic = _sns._topics.get(topic_arn)
+ if not topic:
+ logger.warning("Pipes %s: SNS topic not found %s", pipe.get("Name"), topic_arn)
+ return
+
+ msg_id = new_uuid()
+ message = json.dumps(record)
+ subject = f"Pipes {pipe.get('Name', '')}"
+
+ topic["messages"].append({
+ "id": msg_id,
+ "message": message,
+ "subject": subject,
+ "message_structure": "",
+ "message_attributes": {},
+ "timestamp": int(time.time()),
+ })
+ _sns._fanout(topic_arn, msg_id, message, subject, "", {})
+
+
+def _table_name_from_stream_arn(stream_arn: str) -> str:
+ if "/stream/" not in stream_arn:
+ return ""
+ return stream_arn.split("/stream/", 1)[0].rsplit("/", 1)[-1]
+
+
+def _initial_position(pipe: dict) -> int:
+ from ministack.services import dynamodb as _ddb
+
+ table_name = _table_name_from_stream_arn(pipe.get("Source", ""))
+ if not table_name:
+ return 0
+
+ records = getattr(_ddb, "_stream_records", {}).get(table_name, [])
+ if pipe.get("StartingPosition") == "TRIM_HORIZON":
+ return 0
+ return len(records)
diff --git a/tests/test_cfn.py b/tests/test_cfn.py
index 7df8128a..f93f80a2 100644
--- a/tests/test_cfn.py
+++ b/tests/test_cfn.py
@@ -1392,6 +1392,158 @@ def test_cfn_dynamodb_stream_spec(cfn, ddb):
_wait_stack(cfn, stack_name)
+def test_cfn_pipes_dynamodb_stream_to_sns(cfn, ddb, sqs):
+ uid = _uuid_mod.uuid4().hex[:8]
+ stack_name = f"cfn-pipe-{uid}"
+ table_name = f"cfn-pipe-table-{uid}"
+ queue_name = f"cfn-pipe-q-{uid}"
+ topic_name = f"cfn-pipe-topic-{uid}"
+
+ template = {
+ "AWSTemplateFormatVersion": "2010-09-09",
+ "Resources": {
+ "PipeTable": {
+ "Type": "AWS::DynamoDB::Table",
+ "Properties": {
+ "TableName": table_name,
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "StreamSpecification": {"StreamViewType": "NEW_AND_OLD_IMAGES"},
+ },
+ },
+ "PipeTopic": {
+ "Type": "AWS::SNS::Topic",
+ "Properties": {"TopicName": topic_name},
+ },
+ "PipeQueue": {
+ "Type": "AWS::SQS::Queue",
+ "Properties": {"QueueName": queue_name},
+ },
+ "PipeSubscription": {
+ "Type": "AWS::SNS::Subscription",
+ "Properties": {
+ "Protocol": "sqs",
+ "TopicArn": {"Ref": "PipeTopic"},
+ "Endpoint": {"Fn::GetAtt": ["PipeQueue", "Arn"]},
+ },
+ },
+ "DdbToSnsPipe": {
+ "Type": "AWS::Pipes::Pipe",
+ "Properties": {
+ "Name": f"{stack_name}-pipe",
+ "RoleArn": "arn:aws:iam::000000000000:role/test-pipe-role",
+ "Source": {"Fn::GetAtt": ["PipeTable", "StreamArn"]},
+ "Target": {"Ref": "PipeTopic"},
+ "SourceParameters": {
+ "DynamoDBStreamParameters": {"StartingPosition": "TRIM_HORIZON"}
+ },
+ },
+ },
+ },
+ }
+
+ cfn.create_stack(StackName=stack_name, TemplateBody=json.dumps(template))
+ stack = _wait_stack(cfn, stack_name)
+ assert stack["StackStatus"] == "CREATE_COMPLETE", stack.get("StackStatusReason")
+
+ queue_url = sqs.get_queue_url(QueueName=queue_name)["QueueUrl"]
+
+ ddb.put_item(
+ TableName=table_name,
+ Item={
+ "pk": {"S": "1"},
+ "val": {"S": "hello"},
+ },
+ )
+
+ msg = None
+ deadline = time.time() + 8
+ while time.time() < deadline:
+ out = sqs.receive_message(QueueUrl=queue_url, MaxNumberOfMessages=1, WaitTimeSeconds=1)
+ msgs = out.get("Messages", [])
+ if msgs:
+ msg = msgs[0]
+ break
+
+ assert msg is not None, "Expected DynamoDB stream record to reach SNS/SQS via Pipe"
+
+ envelope = json.loads(msg["Body"])
+ rec = json.loads(envelope["Message"])
+ assert rec.get("eventSource") == "aws:dynamodb"
+ assert rec.get("eventName") in ("INSERT", "MODIFY", "REMOVE")
+
+ dynamodb = rec.get("dynamodb", {})
+ assert dynamodb.get("Keys", {}).get("pk", {}).get("S") == "1"
+ assert dynamodb.get("NewImage", {}).get("pk", {}).get("S") == "1"
+
+ cfn.delete_stack(StackName=stack_name)
+ _wait_stack(cfn, stack_name)
+
+
+def test_cfn_sns_topic_subscription_filter_policy_scope(cfn, sns, sqs):
+ uid = _uuid_mod.uuid4().hex[:8]
+ stack_name = f"cfn-sns-filter-{uid}"
+ queue_name = f"cfn-sns-filter-q-{uid}"
+ topic_name = f"cfn-sns-filter-topic-{uid}"
+
+ template = {
+ "AWSTemplateFormatVersion": "2010-09-09",
+ "Resources": {
+ "FilterQueue": {
+ "Type": "AWS::SQS::Queue",
+ "Properties": {"QueueName": queue_name},
+ },
+ "FilterTopic": {
+ "Type": "AWS::SNS::Topic",
+ "Properties": {
+ "TopicName": topic_name,
+ },
+ },
+ "FilterSubscription": {
+ "Type": "AWS::SNS::Subscription",
+ "Properties": {
+ "Protocol": "sqs",
+ "TopicArn": {"Ref": "FilterTopic"},
+ "Endpoint": {"Fn::GetAtt": ["FilterQueue", "Arn"]},
+ "FilterPolicy": {"color": ["blue"]},
+ },
+ },
+ },
+ "Outputs": {
+ "TopicArn": {"Value": {"Ref": "FilterTopic"}},
+ },
+ }
+
+ cfn.create_stack(StackName=stack_name, TemplateBody=json.dumps(template))
+ stack = _wait_stack(cfn, stack_name)
+ assert stack["StackStatus"] == "CREATE_COMPLETE", stack.get("StackStatusReason")
+
+ outputs = {o["OutputKey"]: o["OutputValue"] for o in stack.get("Outputs", [])}
+ topic_arn = outputs["TopicArn"]
+ queue_url = sqs.get_queue_url(QueueName=queue_name)["QueueUrl"]
+
+ sns.publish(
+ TopicArn=topic_arn,
+ Message="red message",
+ MessageAttributes={"color": {"DataType": "String", "StringValue": "red"}},
+ )
+ msgs = sqs.receive_message(QueueUrl=queue_url, MaxNumberOfMessages=1, WaitTimeSeconds=0)
+ assert len(msgs.get("Messages", [])) == 0
+
+ sns.publish(
+ TopicArn=topic_arn,
+ Message="blue message",
+ MessageAttributes={"color": {"DataType": "String", "StringValue": "blue"}},
+ )
+ msgs = sqs.receive_message(QueueUrl=queue_url, MaxNumberOfMessages=1, WaitTimeSeconds=1)
+ assert len(msgs.get("Messages", [])) == 1
+ body = json.loads(msgs["Messages"][0]["Body"])
+ assert body["Message"] == "blue message"
+
+ cfn.delete_stack(StackName=stack_name)
+ _wait_stack(cfn, stack_name)
+
# ===========================================================================
# CodeBuild Project Tests
# ===========================================================================