Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(pipes): deliver DynamoDB stream records to a Step Functions target (#1521)

Facundo Paredesd72575c27de6
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 57cbd109..d134e81f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,8 @@ Versioning follows [Semantic Versioning](https://semver.org/).
## [Unreleased]
+### Fixed
+- **EventBridge Pipes — a DynamoDB stream reaches a Step Functions target** — the poller delivered to SNS and to nothing else, so a pipe whose target was a state machine reported `RUNNING`, advanced no position and moved no records, with no error and no log line: the CDK `DynamoDbSource` → `StateMachineTarget` pipe every stream-driven workflow uses simply never fired. A `states` target now gets one `StartExecution` per batch, carrying the records as the JSON array `$states.input` reads, which is the shape Pipes delivers on AWS. A pipe's stream position advances only once the whole batch has reached its target, so a `StartExecution` that fails and a target this poller cannot serve (SQS, Lambda, Kinesis, …) both leave the records on the stream for the next poll to retry, and log a warning naming the pipe.
### Added
- **Cognito — `AddCustomAttributes`, and a user pool that reports its schema (`SchemaAttributes`)** — `DescribeUserPool` never returned the pool's attribute schema (the create request's `Schema` was stored under that request-shaped key and no `SchemaAttributes` was ever produced), so a client that diffs its desired schema against the pool saw every attribute as missing. Terraform re-planned an `aws_cognito_user_pool` as changed immediately after creating it and then failed the follow-up apply with `InvalidAction: Unknown Cognito IDP action: AddCustomAttributes`, because that is the operation the provider uses to reconcile added schema items. A pool now carries the full standard attribute set (`sub`, `email`, `email_verified`, `phone_number`, `updated_at`, ... with AWS's data types, mutability, and constraints), a request's `Schema` entries are stored under their `custom:` (or `dev:`) prefix and override a standard entry field-for-field when they redefine one, and `AddCustomAttributes` adds 1-25 attributes per call with `InvalidParameterException` for a duplicate, a standard attribute name, or more than 50 custom attributes, and `ResourceNotFoundException` for an unknown pool. Pools restored from an older snapshot rebuild their schema on load. Also fixed two adjacent read-back gaps that showed up as permanent drift: `EmailConfiguration` now reports `EmailSendingAccount: COGNITO_DEFAULT` when the request omitted it, and `GetUserPoolMfaConfig` no longer invents a disabled `SoftwareTokenMfaConfiguration` for a pool that never configured MFA.
diff --git a/ministack/services/pipes.py b/ministack/services/pipes.py
index d4bee502..0be90b96 100644
--- a/ministack/services/pipes.py
+++ b/ministack/services/pipes.py
@@ -9,7 +9,7 @@ SDK surface:
Runtime (background poller + CloudFormation) scope is intentionally limited to:
- Source: DynamoDB Streams
-- Target: SNS
+- Target: SNS, Step Functions state machine
"""
import copy
@@ -222,39 +222,76 @@ def _poll_once():
account_token = _request_account_id.set(pipe_account_id)
region_token = _request_region.set(pipe_region)
try:
- if pipe.get("CurrentState") != "RUNNING":
- continue
-
- source_arn = pipe.get("Source", "")
- target_arn = pipe.get("Target", "")
- if _arn_service(source_arn) != "dynamodb" or _arn_service(target_arn) != "sns":
- continue
-
- source = _dynamodb_stream_source(source_arn)
- if source is None:
- continue
- source_spec, table_name = source
- if source_spec.account_id != pipe_account_id:
- continue
-
- scope = {"account_id": pipe_account_id, "region": source_spec.region}
- # Positions are absolute stream positions: records expiring off the
- # front of the stream must not shift a pipe's read position.
- horizon = _ddb.stream_start_position(table_name, **scope)
- end = _ddb.stream_end_position(table_name, **scope)
- pos = max(int(_positions.get(pipe["Arn"], 0)), horizon)
- if pos >= end:
- continue
-
- batch = _ddb.stream_records_since(table_name, pos, end - pos, **scope)
- for rec in batch:
- _publish_record_to_sns(target_arn, pipe, rec)
- _positions[pipe["Arn"]] = pos + len(batch)
+ _poll_pipe(_ddb, pipe, pipe_account_id)
finally:
_request_region.reset(region_token)
_request_account_id.reset(account_token)
+def _poll_pipe(_ddb, pipe: dict, pipe_account_id: str) -> None:
+ if pipe.get("CurrentState") != "RUNNING":
+ return
+ if _arn_service(pipe.get("Source", "")) != "dynamodb":
+ return
+
+ source = _dynamodb_stream_source(pipe.get("Source", ""))
+ if source is None:
+ return
+ source_spec, table_name = source
+ if source_spec.account_id != pipe_account_id:
+ return
+
+ scope = {"account_id": pipe_account_id, "region": source_spec.region}
+ # Positions are absolute stream positions: records expiring off the
+ # front of the stream must not shift a pipe's read position.
+ horizon = _ddb.stream_start_position(table_name, **scope)
+ end = _ddb.stream_end_position(table_name, **scope)
+ pos = max(int(_positions.get(pipe["Arn"], 0)), horizon)
+ if pos >= end:
+ return
+
+ batch = _ddb.stream_records_since(table_name, pos, end - pos, **scope)
+ if _deliver_batch(pipe, batch):
+ _positions[pipe["Arn"]] = pos + len(batch)
+
+
+def _deliver_batch(pipe: dict, batch: list) -> bool:
+ """True once every record in the batch has reached the target. A batch that
+ did not is left on the stream: the position stays on it and the next poll
+ retries it, until the records age out of the retention window."""
+ target_arn = pipe.get("Target", "")
+ target_service = _arn_service(target_arn)
+ if target_service == "states":
+ return _start_state_machine_from_records(target_arn, pipe, batch)
+ if target_service == "sns":
+ for rec in batch:
+ _publish_record_to_sns(target_arn, pipe, rec)
+ return True
+ logger.warning(
+ "Pipes %s: holding %d record(s); MiniStack delivers to sns and states, "
+ "not to %s", pipe.get("Name"), len(batch), target_service or target_arn)
+ return False
+
+
+def _start_state_machine_from_records(sm_arn: str, pipe: dict, records: list) -> bool:
+ """Start one execution carrying the batch as the JSON array Pipes delivers.
+
+ `_start_execution` answers the `(status, headers, body)` triple every
+ MiniStack handler answers; anything from 400 up means no execution started.
+ """
+ from ministack.services import stepfunctions as _sfn
+
+ status, _headers, body = _sfn._start_execution({
+ "stateMachineArn": sm_arn,
+ "input": json.dumps(records),
+ })
+ if status >= 400:
+ logger.warning("Pipes %s: StartExecution on %s failed (%s): %s",
+ pipe.get("Name"), sm_arn, status, body)
+ return False
+ return True
+
+
def _publish_record_to_sns(topic_arn: str, pipe: dict, record: dict):
from ministack.services import sns as _sns
diff --git a/tests/test_pipes.py b/tests/test_pipes.py
index 9a8bfd04..969a5248 100644
--- a/tests/test_pipes.py
+++ b/tests/test_pipes.py
@@ -5,7 +5,9 @@ from urllib.parse import quote
import pytest
from ministack.core.responses import (
+ _request_account_id,
_request_region,
+ json_response,
set_request_account_id,
set_request_region,
)
@@ -216,6 +218,123 @@ def test_pipes_dynamodb_stream_read_rejects_cross_account_source(monkeypatch):
_ddb._stream_records.clear()
+def test_pipes_dynamodb_stream_starts_state_machine_with_the_whole_batch(monkeypatch):
+ """A `states` target receives one StartExecution carrying the batch as an array."""
+ from ministack.services import stepfunctions as _sfn
+
+ _pipes.reset()
+ _ddb._stream_records.clear()
+ region_token = _request_region.set("us-east-1")
+ try:
+ table_name = "PipeTable"
+ source_arn = _stream_arn("us-east-1", table_name)
+ target_arn = "arn:aws:states:us-east-1:000000000000:stateMachine:PipeMachine"
+ pipe_arn = "arn:aws:pipes:us-east-1:000000000000:pipe/PipeName"
+ records = [
+ {"eventID": "evt-1", "eventSource": "aws:dynamodb"},
+ {"eventID": "evt-2", "eventSource": "aws:dynamodb"},
+ ]
+
+ _ddb._stream_records.set_scoped("000000000000", "us-east-1", table_name, records)
+ _pipes._pipes["PipeName"] = {
+ "Name": "PipeName",
+ "Arn": pipe_arn,
+ "Source": source_arn,
+ "Target": target_arn,
+ "CurrentState": "RUNNING",
+ "StartingPosition": "TRIM_HORIZON",
+ }
+ _pipes._positions[pipe_arn] = 0
+
+ started = []
+
+ def _record(data):
+ started.append(data)
+ return json_response({"executionArn": "arn:aws:states:x", "startDate": "t"})
+
+ monkeypatch.setattr(_sfn, "_start_execution", _record)
+
+ _pipes._poll_once()
+
+ assert len(started) == 1, "the batch is one execution, not one per record"
+ assert started[0]["stateMachineArn"] == target_arn
+ assert json.loads(started[0]["input"]) == records
+ assert _pipes._positions[pipe_arn] == 2
+ finally:
+ _request_region.reset(region_token)
+ _pipes.reset()
+ _ddb._stream_records.clear()
+
+
+def test_pipes_holds_the_batch_when_the_state_machine_does_not_exist(caplog):
+ """A StartExecution that fails leaves the position on the batch, so the next
+ poll redelivers it instead of skipping the records."""
+ _pipes.reset()
+ _ddb._stream_records.clear()
+ region_token = _request_region.set("us-east-1")
+ account_token = _request_account_id.set("000000000000")
+ try:
+ table_name = "PipeTable"
+ pipe_arn = "arn:aws:pipes:us-east-1:000000000000:pipe/PipeName"
+ records = [{"eventID": "evt-1"}, {"eventID": "evt-2"}]
+ _ddb._stream_records.set_scoped("000000000000", "us-east-1", table_name, records)
+ _pipes._pipes["PipeName"] = {
+ "Name": "PipeName",
+ "Arn": pipe_arn,
+ "Source": _stream_arn("us-east-1", table_name),
+ "Target": "arn:aws:states:us-east-1:000000000000:stateMachine:Missing",
+ "CurrentState": "RUNNING",
+ "StartingPosition": "TRIM_HORIZON",
+ }
+ _pipes._positions[pipe_arn] = 0
+
+ with caplog.at_level("WARNING"):
+ _pipes._poll_once()
+
+ assert "StartExecution" in caplog.text and "PipeName" in caplog.text
+ assert _pipes._positions[pipe_arn] == 0, "an undelivered batch is not skipped"
+ finally:
+ _request_account_id.reset(account_token)
+ _request_region.reset(region_token)
+ _pipes.reset()
+ _ddb._stream_records.clear()
+
+
+def test_pipes_target_this_poller_cannot_deliver_to_says_so(caplog):
+ """A pipe reporting RUNNING while dropping every record names the target it cannot serve."""
+ _pipes.reset()
+ _ddb._stream_records.clear()
+ region_token = _request_region.set("us-east-1")
+ try:
+ table_name = "PipeTable"
+ source_arn = _stream_arn("us-east-1", table_name)
+ target_arn = "arn:aws:sqs:us-east-1:000000000000:PipeQueue"
+ pipe_arn = "arn:aws:pipes:us-east-1:000000000000:pipe/PipeName"
+
+ _ddb._stream_records.set_scoped(
+ "000000000000", "us-east-1", table_name,
+ [{"eventID": "evt-1", "eventSource": "aws:dynamodb"}])
+ _pipes._pipes["PipeName"] = {
+ "Name": "PipeName",
+ "Arn": pipe_arn,
+ "Source": source_arn,
+ "Target": target_arn,
+ "CurrentState": "RUNNING",
+ "StartingPosition": "TRIM_HORIZON",
+ }
+ _pipes._positions[pipe_arn] = 0
+
+ with caplog.at_level("WARNING"):
+ _pipes._poll_once()
+
+ assert "sqs" in caplog.text and "PipeName" in caplog.text
+ assert _pipes._positions[pipe_arn] == 0
+ finally:
+ _request_region.reset(region_token)
+ _pipes.reset()
+ _ddb._stream_records.clear()
+
+
def test_pipes_poller_processes_same_name_pipes_in_each_region(monkeypatch):
from ministack.core.responses import (
get_account_id,