Light, fluffy, and always free - The AWS Local Emulator alternative
feat(pipes): validate ParallelizationFactor on Kinesis and DynamoDB sources (#3192)
Danny5fd89e27bc8b
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/PipesTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/PipesTest.javaindex 9f340226ae..31e5f6dab2 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/PipesTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/PipesTest.java@@ -28,6 +28,10 @@ class PipesTest {return "arn:aws:sqs:" + REGION + ":" + ACCOUNT_ID + ":" + queueName;}+ private static String kinesisArn(String streamName) {+ return "arn:aws:kinesis:" + REGION + ":" + ACCOUNT_ID + ":stream/" + streamName;+ }+@BeforeAllstatic void setup() {pipes = TestFixtures.pipesClient();@@ -344,4 +348,61 @@ class PipesTest {try { sqs.deleteQueue(DeleteQueueRequest.builder().queueUrl(nfTgtUrl).build()); } catch (Exception ignored) {}}}++ @Test+ @Order(14)+ void parallelizationFactorRoundTripsOnKinesisSource() {+ String pfPipeName = TestFixtures.uniqueName("pipe-pf");+ String pfStream = TestFixtures.uniqueName("pipe-pf-stream");++ try {+ pipes.createPipe(CreatePipeRequest.builder()+ .name(pfPipeName)+ .source(kinesisArn(pfStream))+ .target(sqsArn(tgtQueue))+ .roleArn(ROLE_ARN)+ .desiredState(RequestedPipeState.STOPPED)+ .sourceParameters(PipeSourceParameters.builder()+ .kinesisStreamParameters(PipeSourceKinesisStreamParameters.builder()+ .startingPosition(KinesisStreamStartPosition.TRIM_HORIZON)+ .parallelizationFactor(4)+ .build())+ .build())+ .build());++ DescribePipeResponse response = pipes.describePipe(DescribePipeRequest.builder()+ .name(pfPipeName).build());++ assertThat(response.sourceParameters().kinesisStreamParameters().parallelizationFactor())+ .isEqualTo(4);+ } finally {+ try { pipes.deletePipe(DeletePipeRequest.builder().name(pfPipeName).build()); } catch (Exception ignored) {}+ }+ }++ @Test+ @Order(15)+ void parallelizationFactorOutOfRangeIsRejected() {+ String pfPipeName = TestFixtures.uniqueName("pipe-pf-invalid");+ String pfStream = TestFixtures.uniqueName("pipe-pf-invalid-stream");++ assertThatThrownBy(() -> pipes.createPipe(CreatePipeRequest.builder()+ .name(pfPipeName)+ .source(kinesisArn(pfStream))+ .target(sqsArn(tgtQueue))+ .roleArn(ROLE_ARN)+ .desiredState(RequestedPipeState.STOPPED)+ .sourceParameters(PipeSourceParameters.builder()+ .kinesisStreamParameters(PipeSourceKinesisStreamParameters.builder()+ .startingPosition(KinesisStreamStartPosition.TRIM_HORIZON)+ .parallelizationFactor(11)+ .build())+ .build())+ .build()))+ .isInstanceOf(ValidationException.class);++ assertThatThrownBy(() -> pipes.describePipe(DescribePipeRequest.builder()+ .name(pfPipeName).build()))+ .isInstanceOf(NotFoundException.class);+ }}diff --git a/docs/services/pipes.md b/docs/services/pipes.mdindex c143c2b458..fd87ff2f5b 100644--- a/docs/services/pipes.md+++ b/docs/services/pipes.md@@ -95,6 +95,34 @@ Floci emulates EventBridge Pipes with the following supported source and target- Kinesis streams- Step Functions state machines+## ParallelizationFactor++`CreatePipe` and `UpdatePipe` accept a `ParallelizationFactor` integer between 1 and 10 on the+`KinesisStreamParameters` and `DynamoDBStreamParameters` source blocks, matching the AWS wire+format. `DescribePipe` echoes it back as part of `SourceParameters`. `ListPipes` returns pipe+summaries only and omits `SourceParameters` entirely, matching AWS.++```bash+aws pipes create-pipe \+ --name kinesis-pipe \+ --source "arn:aws:kinesis:us-east-1:000000000000:stream/events" \+ --target "arn:aws:lambda:us-east-1:000000000000:function:my-function" \+ --role-arn "arn:aws:iam::000000000000:role/pipe-role" \+ --source-parameters '{"KinesisStreamParameters":{"StartingPosition":"TRIM_HORIZON","ParallelizationFactor":4}}' \+ --endpoint-url $AWS_ENDPOINT_URL+```++Validation mirrors AWS: values outside 1 to 10 are rejected with `ValidationException`, and so is+the field on a source its parameter block does not describe, for example+`KinesisStreamParameters.ParallelizationFactor` on an SQS source.++!!! note "Enforcement status"+ The configured `ParallelizationFactor` is persisted and returned on the wire, but the poller+ does not yet process concurrent batches per shard. Floci opens an iterator on a single shard+ (`shardId-000000000000`) per Kinesis or DynamoDB Streams pipe and delivers one batch at a time+ regardless of the configured value. Multi-shard polling and real per-shard concurrency are+ tracked as follow-ups.+## EnrichmentA pipe's optional enrichment step (`source → filter → enrichment → target`) is emulated fordiff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.javaindex c6e5c7e2a8..42a8021e2d 100644--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java@@ -280,6 +280,11 @@ public class PipesPoller implements Resettable {}}+ /**+ * Opens an iterator on the stream's first shard. Floci polls that one shard and delivers one+ * batch at a time, so a pipe's {@code ParallelizationFactor} is persisted and returned on the+ * wire but not enforced here: see docs/services/pipes.md.+ */private String initKinesisIterator(String streamName, String region, String accountId) {try {return (accountId != null)@@ -506,6 +511,10 @@ public class PipesPoller implements Resettable {return failed;}+ /**+ * Opens an iterator on the stream's first shard, under the same single-shard limit as+ * {@link #initKinesisIterator}.+ */private String initDynamoDbIterator(String streamArn) {try {return dynamoDbStreamService.getShardIterator(diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.javaindex 82c8ec02d6..319184dd70 100644--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.java+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.java@@ -33,6 +33,9 @@ public class PipesService implements TagHandler, ResourceProvider {private static final Logger LOG = Logger.getLogger(PipesService.class);+ private static final int MIN_PARALLELIZATION_FACTOR = 1;+ private static final int MAX_PARALLELIZATION_FACTOR = 10;+private final StorageBackend<String, Pipe> storage;private final RegionResolver regionResolver;private final PipesPoller poller;@@ -324,11 +327,10 @@ public class PipesService implements TagHandler, ResourceProvider {}if (source.startsWith("smk://")) {requireKafkaParameters(sourceParameters, "SelfManagedKafkaParameters");- return;- }- if (source.contains(":kafka:")) {+ } else if (source.contains(":kafka:")) {requireKafkaParameters(sourceParameters, "ManagedStreamingKafkaParameters");}+ validateParallelizationFactor(source, sourceParameters);}private void requireKafkaParameters(JsonNode sourceParameters, String parameterBlock) {@@ -342,4 +344,49 @@ public class PipesService implements TagHandler, ResourceProvider {"SourceParameters." + parameterBlock + ".TopicName is required", 400);}}++ /**+ * Applies the AWS bounds on {@code ParallelizationFactor}. It is a member of the Kinesis and+ * DynamoDB Stream parameter blocks only, so a value carried by a block that does not describe+ * the pipe's source is rejected rather than silently kept.+ */+ private void validateParallelizationFactor(String source, JsonNode sourceParameters) {+ if (sourceParameters == null) {+ return;+ }+ validateParallelizationFactorBlock(sourceParameters, "KinesisStreamParameters",+ "Kinesis stream", source.contains(":kinesis:"));+ validateParallelizationFactorBlock(sourceParameters, "DynamoDBStreamParameters",+ "DynamoDB Stream", source.contains(":dynamodb:"));+ }++ private void validateParallelizationFactorBlock(JsonNode sourceParameters, String parameterBlock,+ String sourceDescription, boolean sourceMatchesBlock) {+ JsonNode factor = sourceParameters.path(parameterBlock).path("ParallelizationFactor");+ if (factor.isMissingNode() || factor.isNull()) {+ return;+ }+ String property = "SourceParameters." + parameterBlock + ".ParallelizationFactor";+ if (!sourceMatchesBlock) {+ throw new AwsException("ValidationException",+ property + " is only supported for " + sourceDescription + " sources", 400);+ }+ if (!factor.isNumber()) {+ throw new AwsException("ValidationException",+ property + " must be a numeric value", 400);+ }+ if (!factor.isIntegralNumber()) {+ throw new AwsException("ValidationException",+ property + " must be an integer", 400);+ }+ // canConvertToInt keeps a value wider than an int out of the bounds check: asLong would+ // hand back the low 64 bits of a BigInteger, so 2^64 + 5 would read as 5 and be accepted.+ if (!factor.canConvertToInt()+ || factor.intValue() < MIN_PARALLELIZATION_FACTOR+ || factor.intValue() > MAX_PARALLELIZATION_FACTOR) {+ throw new AwsException("ValidationException",+ property + " must be between " + MIN_PARALLELIZATION_FACTOR + " and "+ + MAX_PARALLELIZATION_FACTOR + " (got " + factor.asText() + ")", 400);+ }+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesIntegrationTest.javaindex a8a80a57b2..9e72655869 100644--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesIntegrationTest.java@@ -252,4 +252,89 @@ class PipesIntegrationTest {.then().statusCode(400);}++ @Test+ @Order(14)+ void createPipeReturnsParallelizationFactorInResponse() {+ given()+ .contentType("application/json")+ .body("""+ {+ "Source": "arn:aws:kinesis:us-east-1:000000000000:stream/pf-stream",+ "Target": "arn:aws:sqs:us-east-1:000000000000:pf-target",+ "RoleArn": "arn:aws:iam::000000000000:role/pipe-role",+ "DesiredState": "STOPPED",+ "SourceParameters": {+ "KinesisStreamParameters": {+ "StartingPosition": "TRIM_HORIZON",+ "ParallelizationFactor": 4+ }+ }+ }+ """)+ .when()+ .post("/v1/pipes/pf-pipe")+ .then()+ .statusCode(200)+ .body("SourceParameters.KinesisStreamParameters.ParallelizationFactor", equalTo(4));++ given()+ .contentType("application/json")+ .when()+ .get("/v1/pipes/pf-pipe")+ .then()+ .statusCode(200)+ .body("SourceParameters.KinesisStreamParameters.ParallelizationFactor", equalTo(4));++ given()+ .contentType("application/json")+ .when()+ .delete("/v1/pipes/pf-pipe")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(15)+ void createPipeParallelizationFactorOutOfRangeReturns400() {+ given()+ .contentType("application/json")+ .body("""+ {+ "Source": "arn:aws:kinesis:us-east-1:000000000000:stream/pf-stream",+ "Target": "arn:aws:sqs:us-east-1:000000000000:pf-target",+ "RoleArn": "arn:aws:iam::000000000000:role/pipe-role",+ "DesiredState": "STOPPED",+ "SourceParameters": {+ "KinesisStreamParameters": {"ParallelizationFactor": 11}+ }+ }+ """)+ .when()+ .post("/v1/pipes/pf-invalid-pipe")+ .then()+ .statusCode(400);+ }++ @Test+ @Order(16)+ void createPipeParallelizationFactorOnSqsSourceReturns400() {+ given()+ .contentType("application/json")+ .body("""+ {+ "Source": "arn:aws:sqs:us-east-1:000000000000:pf-source",+ "Target": "arn:aws:sqs:us-east-1:000000000000:pf-target",+ "RoleArn": "arn:aws:iam::000000000000:role/pipe-role",+ "DesiredState": "STOPPED",+ "SourceParameters": {+ "KinesisStreamParameters": {"ParallelizationFactor": 4}+ }+ }+ """)+ .when()+ .post("/v1/pipes/pf-wrong-source-pipe")+ .then()+ .statusCode(400);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.javaindex 669e5687b0..e47fc9303d 100644--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.java@@ -1,5 +1,7 @@package io.github.hectorvent.floci.services.pipes;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.InMemoryStorage;@@ -21,6 +23,8 @@ import static org.mockito.Mockito.when;class PipesServiceTest {+ private static final ObjectMapper MAPPER = new ObjectMapper();+private PipesService pipesService;@BeforeEach@@ -355,4 +359,159 @@ class PipesServiceTest {pipesService.describePipe("region-pipe", "eu-west-1"));assertEquals("NotFoundException", ex.getErrorCode());}++ @Test+ void createPipeAcceptsParallelizationFactorOnKinesisSource() {+ Pipe pipe = pipesService.createPipe("kinesis-pf-pipe",+ "arn:aws:kinesis:us-east-1:000000000000:stream/events",+ "arn:aws:sqs:us-east-1:000000000000:target",+ "arn:aws:iam::000000000000:role/role",+ null, null, null,+ sourceParameters("""+ {"KinesisStreamParameters":{"StartingPosition":"TRIM_HORIZON","ParallelizationFactor":4}}"""),+ null, null, null, "us-east-1");++ assertEquals(4, pipe.getSourceParameters()+ .path("KinesisStreamParameters").path("ParallelizationFactor").asInt());+ }++ @Test+ void createPipeAcceptsParallelizationFactorOnDynamoDbStreamSource() {+ Pipe pipe = pipesService.createPipe("ddb-pf-pipe",+ "arn:aws:dynamodb:us-east-1:000000000000:table/orders/stream/2024-01-01T00:00:00.000",+ "arn:aws:sqs:us-east-1:000000000000:target",+ "arn:aws:iam::000000000000:role/role",+ null, null, null,+ sourceParameters("""+ {"DynamoDBStreamParameters":{"StartingPosition":"LATEST","ParallelizationFactor":10}}"""),+ null, null, null, "us-east-1");++ assertEquals(10, pipe.getSourceParameters()+ .path("DynamoDBStreamParameters").path("ParallelizationFactor").asInt());+ }++ @Test+ void createPipeRejectsParallelizationFactorAboveMaximum() {+ AwsException ex = assertThrows(AwsException.class, () ->+ pipesService.createPipe("pf-too-high",+ "arn:aws:kinesis:us-east-1:000000000000:stream/events",+ "arn:aws:sqs:us-east-1:000000000000:target",+ "arn:aws:iam::000000000000:role/role",+ null, null, null,+ sourceParameters("""+ {"KinesisStreamParameters":{"ParallelizationFactor":11}}"""),+ null, null, null, "us-east-1"));++ assertEquals("ValidationException", ex.getErrorCode());+ assertTrue(ex.getMessage().contains("must be between 1 and 10"), ex.getMessage());+ }++ @Test+ void createPipeRejectsParallelizationFactorBelowMinimum() {+ AwsException ex = assertThrows(AwsException.class, () ->+ pipesService.createPipe("pf-too-low",+ "arn:aws:kinesis:us-east-1:000000000000:stream/events",+ "arn:aws:sqs:us-east-1:000000000000:target",+ "arn:aws:iam::000000000000:role/role",+ null, null, null,+ sourceParameters("""+ {"KinesisStreamParameters":{"ParallelizationFactor":0}}"""),+ null, null, null, "us-east-1"));++ assertEquals("ValidationException", ex.getErrorCode());+ assertTrue(ex.getMessage().contains("must be between 1 and 10"), ex.getMessage());+ }++ @Test+ void createPipeRejectsFractionalParallelizationFactor() {+ AwsException ex = assertThrows(AwsException.class, () ->+ pipesService.createPipe("pf-fractional",+ "arn:aws:kinesis:us-east-1:000000000000:stream/events",+ "arn:aws:sqs:us-east-1:000000000000:target",+ "arn:aws:iam::000000000000:role/role",+ null, null, null,+ sourceParameters("""+ {"KinesisStreamParameters":{"ParallelizationFactor":2.5}}"""),+ null, null, null, "us-east-1"));++ assertEquals("ValidationException", ex.getErrorCode());+ assertTrue(ex.getMessage().contains("must be an integer"), ex.getMessage());+ }++ @Test+ void createPipeRejectsParallelizationFactorWiderThanAnInt() {+ AwsException ex = assertThrows(AwsException.class, () ->+ pipesService.createPipe("pf-oversized",+ "arn:aws:kinesis:us-east-1:000000000000:stream/events",+ "arn:aws:sqs:us-east-1:000000000000:target",+ "arn:aws:iam::000000000000:role/role",+ null, null, null,+ sourceParameters("""+ {"KinesisStreamParameters":{"ParallelizationFactor":18446744073709551621}}"""),+ null, null, null, "us-east-1"));++ assertEquals("ValidationException", ex.getErrorCode());+ assertTrue(ex.getMessage().contains("18446744073709551621"), ex.getMessage());+ }++ @Test+ void createPipeRejectsNonNumericParallelizationFactor() {+ AwsException ex = assertThrows(AwsException.class, () ->+ pipesService.createPipe("pf-non-numeric",+ "arn:aws:kinesis:us-east-1:000000000000:stream/events",+ "arn:aws:sqs:us-east-1:000000000000:target",+ "arn:aws:iam::000000000000:role/role",+ null, null, null,+ sourceParameters("""+ {"KinesisStreamParameters":{"ParallelizationFactor":"4"}}"""),+ null, null, null, "us-east-1"));++ assertEquals("ValidationException", ex.getErrorCode());+ assertTrue(ex.getMessage().contains("must be a numeric value"), ex.getMessage());+ }++ @Test+ void createPipeRejectsParallelizationFactorOnSqsSource() {+ AwsException ex = assertThrows(AwsException.class, () ->+ pipesService.createPipe("pf-wrong-source",+ "arn:aws:sqs:us-east-1:000000000000:source",+ "arn:aws:sqs:us-east-1:000000000000:target",+ "arn:aws:iam::000000000000:role/role",+ null, null, null,+ sourceParameters("""+ {"KinesisStreamParameters":{"ParallelizationFactor":4}}"""),+ null, null, null, "us-east-1"));++ assertEquals("ValidationException", ex.getErrorCode());+ assertTrue(ex.getMessage().contains("only supported for Kinesis stream sources"), ex.getMessage());+ }++ @Test+ void updatePipeRejectsInvalidParallelizationFactor() {+ pipesService.createPipe("pf-update-pipe",+ "arn:aws:kinesis:us-east-1:000000000000:stream/events",+ "arn:aws:sqs:us-east-1:000000000000:target",+ "arn:aws:iam::000000000000:role/role",+ null, null, null,+ sourceParameters("""+ {"KinesisStreamParameters":{"ParallelizationFactor":2}}"""),+ null, null, null, "us-east-1");++ AwsException ex = assertThrows(AwsException.class, () ->+ pipesService.updatePipe("pf-update-pipe", null, null, null, null, null,+ sourceParameters("""+ {"KinesisStreamParameters":{"ParallelizationFactor":99}}"""),+ null, null, "us-east-1"));++ assertEquals("ValidationException", ex.getErrorCode());+ assertTrue(ex.getMessage().contains("must be between 1 and 10"), ex.getMessage());+ }++ private static JsonNode sourceParameters(String json) {+ try {+ return MAPPER.readTree(json);+ } catch (Exception e) {+ throw new IllegalStateException("invalid test JSON: " + json, e);+ }+ }}