Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(pipes): deliver DynamoDB stream records to a Step Functions target (#1521)
Facundo Paredesd72575c27de6
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 57cbd109..d134e81f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,8 @@ Versioning follows [Semantic Versioning](https://semver.org/).## [Unreleased]+### Fixed+- **EventBridge Pipes — a DynamoDB stream reaches a Step Functions target** — the poller delivered to SNS and to nothing else, so a pipe whose target was a state machine reported `RUNNING`, advanced no position and moved no records, with no error and no log line: the CDK `DynamoDbSource` → `StateMachineTarget` pipe every stream-driven workflow uses simply never fired. A `states` target now gets one `StartExecution` per batch, carrying the records as the JSON array `$states.input` reads, which is the shape Pipes delivers on AWS. A pipe's stream position advances only once the whole batch has reached its target, so a `StartExecution` that fails and a target this poller cannot serve (SQS, Lambda, Kinesis, …) both leave the records on the stream for the next poll to retry, and log a warning naming the pipe.### Added- **Cognito — `AddCustomAttributes`, and a user pool that reports its schema (`SchemaAttributes`)** — `DescribeUserPool` never returned the pool's attribute schema (the create request's `Schema` was stored under that request-shaped key and no `SchemaAttributes` was ever produced), so a client that diffs its desired schema against the pool saw every attribute as missing. Terraform re-planned an `aws_cognito_user_pool` as changed immediately after creating it and then failed the follow-up apply with `InvalidAction: Unknown Cognito IDP action: AddCustomAttributes`, because that is the operation the provider uses to reconcile added schema items. A pool now carries the full standard attribute set (`sub`, `email`, `email_verified`, `phone_number`, `updated_at`, ... with AWS's data types, mutability, and constraints), a request's `Schema` entries are stored under their `custom:` (or `dev:`) prefix and override a standard entry field-for-field when they redefine one, and `AddCustomAttributes` adds 1-25 attributes per call with `InvalidParameterException` for a duplicate, a standard attribute name, or more than 50 custom attributes, and `ResourceNotFoundException` for an unknown pool. Pools restored from an older snapshot rebuild their schema on load. Also fixed two adjacent read-back gaps that showed up as permanent drift: `EmailConfiguration` now reports `EmailSendingAccount: COGNITO_DEFAULT` when the request omitted it, and `GetUserPoolMfaConfig` no longer invents a disabled `SoftwareTokenMfaConfiguration` for a pool that never configured MFA.diff --git a/ministack/services/pipes.py b/ministack/services/pipes.pyindex d4bee502..0be90b96 100644--- a/ministack/services/pipes.py+++ b/ministack/services/pipes.py@@ -9,7 +9,7 @@ SDK surface:Runtime (background poller + CloudFormation) scope is intentionally limited to:- Source: DynamoDB Streams-- Target: SNS+- Target: SNS, Step Functions state machine"""import copy@@ -222,39 +222,76 @@ def _poll_once():account_token = _request_account_id.set(pipe_account_id)region_token = _request_region.set(pipe_region)try:- if pipe.get("CurrentState") != "RUNNING":- continue-- source_arn = pipe.get("Source", "")- target_arn = pipe.get("Target", "")- if _arn_service(source_arn) != "dynamodb" or _arn_service(target_arn) != "sns":- continue-- source = _dynamodb_stream_source(source_arn)- if source is None:- continue- source_spec, table_name = source- if source_spec.account_id != pipe_account_id:- continue-- scope = {"account_id": pipe_account_id, "region": source_spec.region}- # Positions are absolute stream positions: records expiring off the- # front of the stream must not shift a pipe's read position.- horizon = _ddb.stream_start_position(table_name, **scope)- end = _ddb.stream_end_position(table_name, **scope)- pos = max(int(_positions.get(pipe["Arn"], 0)), horizon)- if pos >= end:- continue-- batch = _ddb.stream_records_since(table_name, pos, end - pos, **scope)- for rec in batch:- _publish_record_to_sns(target_arn, pipe, rec)- _positions[pipe["Arn"]] = pos + len(batch)+ _poll_pipe(_ddb, pipe, pipe_account_id)finally:_request_region.reset(region_token)_request_account_id.reset(account_token)+def _poll_pipe(_ddb, pipe: dict, pipe_account_id: str) -> None:+ if pipe.get("CurrentState") != "RUNNING":+ return+ if _arn_service(pipe.get("Source", "")) != "dynamodb":+ return++ source = _dynamodb_stream_source(pipe.get("Source", ""))+ if source is None:+ return+ source_spec, table_name = source+ if source_spec.account_id != pipe_account_id:+ return++ scope = {"account_id": pipe_account_id, "region": source_spec.region}+ # Positions are absolute stream positions: records expiring off the+ # front of the stream must not shift a pipe's read position.+ horizon = _ddb.stream_start_position(table_name, **scope)+ end = _ddb.stream_end_position(table_name, **scope)+ pos = max(int(_positions.get(pipe["Arn"], 0)), horizon)+ if pos >= end:+ return++ batch = _ddb.stream_records_since(table_name, pos, end - pos, **scope)+ if _deliver_batch(pipe, batch):+ _positions[pipe["Arn"]] = pos + len(batch)+++def _deliver_batch(pipe: dict, batch: list) -> bool:+ """True once every record in the batch has reached the target. A batch that+ did not is left on the stream: the position stays on it and the next poll+ retries it, until the records age out of the retention window."""+ target_arn = pipe.get("Target", "")+ target_service = _arn_service(target_arn)+ if target_service == "states":+ return _start_state_machine_from_records(target_arn, pipe, batch)+ if target_service == "sns":+ for rec in batch:+ _publish_record_to_sns(target_arn, pipe, rec)+ return True+ logger.warning(+ "Pipes %s: holding %d record(s); MiniStack delivers to sns and states, "+ "not to %s", pipe.get("Name"), len(batch), target_service or target_arn)+ return False+++def _start_state_machine_from_records(sm_arn: str, pipe: dict, records: list) -> bool:+ """Start one execution carrying the batch as the JSON array Pipes delivers.++ `_start_execution` answers the `(status, headers, body)` triple every+ MiniStack handler answers; anything from 400 up means no execution started.+ """+ from ministack.services import stepfunctions as _sfn++ status, _headers, body = _sfn._start_execution({+ "stateMachineArn": sm_arn,+ "input": json.dumps(records),+ })+ if status >= 400:+ logger.warning("Pipes %s: StartExecution on %s failed (%s): %s",+ pipe.get("Name"), sm_arn, status, body)+ return False+ return True++def _publish_record_to_sns(topic_arn: str, pipe: dict, record: dict):from ministack.services import sns as _snsdiff --git a/tests/test_pipes.py b/tests/test_pipes.pyindex 9a8bfd04..969a5248 100644--- a/tests/test_pipes.py+++ b/tests/test_pipes.py@@ -5,7 +5,9 @@ from urllib.parse import quoteimport pytestfrom ministack.core.responses import (+ _request_account_id,_request_region,+ json_response,set_request_account_id,set_request_region,)@@ -216,6 +218,123 @@ def test_pipes_dynamodb_stream_read_rejects_cross_account_source(monkeypatch):_ddb._stream_records.clear()+def test_pipes_dynamodb_stream_starts_state_machine_with_the_whole_batch(monkeypatch):+ """A `states` target receives one StartExecution carrying the batch as an array."""+ from ministack.services import stepfunctions as _sfn++ _pipes.reset()+ _ddb._stream_records.clear()+ region_token = _request_region.set("us-east-1")+ try:+ table_name = "PipeTable"+ source_arn = _stream_arn("us-east-1", table_name)+ target_arn = "arn:aws:states:us-east-1:000000000000:stateMachine:PipeMachine"+ pipe_arn = "arn:aws:pipes:us-east-1:000000000000:pipe/PipeName"+ records = [+ {"eventID": "evt-1", "eventSource": "aws:dynamodb"},+ {"eventID": "evt-2", "eventSource": "aws:dynamodb"},+ ]++ _ddb._stream_records.set_scoped("000000000000", "us-east-1", table_name, records)+ _pipes._pipes["PipeName"] = {+ "Name": "PipeName",+ "Arn": pipe_arn,+ "Source": source_arn,+ "Target": target_arn,+ "CurrentState": "RUNNING",+ "StartingPosition": "TRIM_HORIZON",+ }+ _pipes._positions[pipe_arn] = 0++ started = []++ def _record(data):+ started.append(data)+ return json_response({"executionArn": "arn:aws:states:x", "startDate": "t"})++ monkeypatch.setattr(_sfn, "_start_execution", _record)++ _pipes._poll_once()++ assert len(started) == 1, "the batch is one execution, not one per record"+ assert started[0]["stateMachineArn"] == target_arn+ assert json.loads(started[0]["input"]) == records+ assert _pipes._positions[pipe_arn] == 2+ finally:+ _request_region.reset(region_token)+ _pipes.reset()+ _ddb._stream_records.clear()+++def test_pipes_holds_the_batch_when_the_state_machine_does_not_exist(caplog):+ """A StartExecution that fails leaves the position on the batch, so the next+ poll redelivers it instead of skipping the records."""+ _pipes.reset()+ _ddb._stream_records.clear()+ region_token = _request_region.set("us-east-1")+ account_token = _request_account_id.set("000000000000")+ try:+ table_name = "PipeTable"+ pipe_arn = "arn:aws:pipes:us-east-1:000000000000:pipe/PipeName"+ records = [{"eventID": "evt-1"}, {"eventID": "evt-2"}]+ _ddb._stream_records.set_scoped("000000000000", "us-east-1", table_name, records)+ _pipes._pipes["PipeName"] = {+ "Name": "PipeName",+ "Arn": pipe_arn,+ "Source": _stream_arn("us-east-1", table_name),+ "Target": "arn:aws:states:us-east-1:000000000000:stateMachine:Missing",+ "CurrentState": "RUNNING",+ "StartingPosition": "TRIM_HORIZON",+ }+ _pipes._positions[pipe_arn] = 0++ with caplog.at_level("WARNING"):+ _pipes._poll_once()++ assert "StartExecution" in caplog.text and "PipeName" in caplog.text+ assert _pipes._positions[pipe_arn] == 0, "an undelivered batch is not skipped"+ finally:+ _request_account_id.reset(account_token)+ _request_region.reset(region_token)+ _pipes.reset()+ _ddb._stream_records.clear()+++def test_pipes_target_this_poller_cannot_deliver_to_says_so(caplog):+ """A pipe reporting RUNNING while dropping every record names the target it cannot serve."""+ _pipes.reset()+ _ddb._stream_records.clear()+ region_token = _request_region.set("us-east-1")+ try:+ table_name = "PipeTable"+ source_arn = _stream_arn("us-east-1", table_name)+ target_arn = "arn:aws:sqs:us-east-1:000000000000:PipeQueue"+ pipe_arn = "arn:aws:pipes:us-east-1:000000000000:pipe/PipeName"++ _ddb._stream_records.set_scoped(+ "000000000000", "us-east-1", table_name,+ [{"eventID": "evt-1", "eventSource": "aws:dynamodb"}])+ _pipes._pipes["PipeName"] = {+ "Name": "PipeName",+ "Arn": pipe_arn,+ "Source": source_arn,+ "Target": target_arn,+ "CurrentState": "RUNNING",+ "StartingPosition": "TRIM_HORIZON",+ }+ _pipes._positions[pipe_arn] = 0++ with caplog.at_level("WARNING"):+ _pipes._poll_once()++ assert "sqs" in caplog.text and "PipeName" in caplog.text+ assert _pipes._positions[pipe_arn] == 0+ finally:+ _request_region.reset(region_token)+ _pipes.reset()+ _ddb._stream_records.clear()++def test_pipes_poller_processes_same_name_pipes_in_each_region(monkeypatch):from ministack.core.responses import (get_account_id,