Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(pipes): validate ParallelizationFactor on Kinesis and DynamoDB sources (#3192)

Danny5fd89e27bc8b
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/PipesTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/PipesTest.java
index 9f340226ae..31e5f6dab2 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/PipesTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/PipesTest.java
@@ -28,6 +28,10 @@ class PipesTest {
return "arn:aws:sqs:" + REGION + ":" + ACCOUNT_ID + ":" + queueName;
}
+ private static String kinesisArn(String streamName) {
+ return "arn:aws:kinesis:" + REGION + ":" + ACCOUNT_ID + ":stream/" + streamName;
+ }
+
@BeforeAll
static void setup() {
pipes = TestFixtures.pipesClient();
@@ -344,4 +348,61 @@ class PipesTest {
try { sqs.deleteQueue(DeleteQueueRequest.builder().queueUrl(nfTgtUrl).build()); } catch (Exception ignored) {}
}
}
+
+ @Test
+ @Order(14)
+ void parallelizationFactorRoundTripsOnKinesisSource() {
+ String pfPipeName = TestFixtures.uniqueName("pipe-pf");
+ String pfStream = TestFixtures.uniqueName("pipe-pf-stream");
+
+ try {
+ pipes.createPipe(CreatePipeRequest.builder()
+ .name(pfPipeName)
+ .source(kinesisArn(pfStream))
+ .target(sqsArn(tgtQueue))
+ .roleArn(ROLE_ARN)
+ .desiredState(RequestedPipeState.STOPPED)
+ .sourceParameters(PipeSourceParameters.builder()
+ .kinesisStreamParameters(PipeSourceKinesisStreamParameters.builder()
+ .startingPosition(KinesisStreamStartPosition.TRIM_HORIZON)
+ .parallelizationFactor(4)
+ .build())
+ .build())
+ .build());
+
+ DescribePipeResponse response = pipes.describePipe(DescribePipeRequest.builder()
+ .name(pfPipeName).build());
+
+ assertThat(response.sourceParameters().kinesisStreamParameters().parallelizationFactor())
+ .isEqualTo(4);
+ } finally {
+ try { pipes.deletePipe(DeletePipeRequest.builder().name(pfPipeName).build()); } catch (Exception ignored) {}
+ }
+ }
+
+ @Test
+ @Order(15)
+ void parallelizationFactorOutOfRangeIsRejected() {
+ String pfPipeName = TestFixtures.uniqueName("pipe-pf-invalid");
+ String pfStream = TestFixtures.uniqueName("pipe-pf-invalid-stream");
+
+ assertThatThrownBy(() -> pipes.createPipe(CreatePipeRequest.builder()
+ .name(pfPipeName)
+ .source(kinesisArn(pfStream))
+ .target(sqsArn(tgtQueue))
+ .roleArn(ROLE_ARN)
+ .desiredState(RequestedPipeState.STOPPED)
+ .sourceParameters(PipeSourceParameters.builder()
+ .kinesisStreamParameters(PipeSourceKinesisStreamParameters.builder()
+ .startingPosition(KinesisStreamStartPosition.TRIM_HORIZON)
+ .parallelizationFactor(11)
+ .build())
+ .build())
+ .build()))
+ .isInstanceOf(ValidationException.class);
+
+ assertThatThrownBy(() -> pipes.describePipe(DescribePipeRequest.builder()
+ .name(pfPipeName).build()))
+ .isInstanceOf(NotFoundException.class);
+ }
}
diff --git a/docs/services/pipes.md b/docs/services/pipes.md
index c143c2b458..fd87ff2f5b 100644
--- a/docs/services/pipes.md
+++ b/docs/services/pipes.md
@@ -95,6 +95,34 @@ Floci emulates EventBridge Pipes with the following supported source and target
- Kinesis streams
- Step Functions state machines
+## ParallelizationFactor
+
+`CreatePipe` and `UpdatePipe` accept a `ParallelizationFactor` integer between 1 and 10 on the
+`KinesisStreamParameters` and `DynamoDBStreamParameters` source blocks, matching the AWS wire
+format. `DescribePipe` echoes it back as part of `SourceParameters`. `ListPipes` returns pipe
+summaries only and omits `SourceParameters` entirely, matching AWS.
+
+```bash
+aws pipes create-pipe \
+ --name kinesis-pipe \
+ --source "arn:aws:kinesis:us-east-1:000000000000:stream/events" \
+ --target "arn:aws:lambda:us-east-1:000000000000:function:my-function" \
+ --role-arn "arn:aws:iam::000000000000:role/pipe-role" \
+ --source-parameters '{"KinesisStreamParameters":{"StartingPosition":"TRIM_HORIZON","ParallelizationFactor":4}}' \
+ --endpoint-url $AWS_ENDPOINT_URL
+```
+
+Validation mirrors AWS: values outside 1 to 10 are rejected with `ValidationException`, and so is
+the field on a source its parameter block does not describe, for example
+`KinesisStreamParameters.ParallelizationFactor` on an SQS source.
+
+!!! note "Enforcement status"
+ The configured `ParallelizationFactor` is persisted and returned on the wire, but the poller
+ does not yet process concurrent batches per shard. Floci opens an iterator on a single shard
+ (`shardId-000000000000`) per Kinesis or DynamoDB Streams pipe and delivers one batch at a time
+ regardless of the configured value. Multi-shard polling and real per-shard concurrency are
+ tracked as follow-ups.
+
## Enrichment
A pipe's optional enrichment step (`source → filter → enrichment → target`) is emulated for
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
index c6e5c7e2a8..42a8021e2d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
@@ -280,6 +280,11 @@ public class PipesPoller implements Resettable {
}
}
+ /**
+ * Opens an iterator on the stream's first shard. Floci polls that one shard and delivers one
+ * batch at a time, so a pipe's {@code ParallelizationFactor} is persisted and returned on the
+ * wire but not enforced here: see docs/services/pipes.md.
+ */
private String initKinesisIterator(String streamName, String region, String accountId) {
try {
return (accountId != null)
@@ -506,6 +511,10 @@ public class PipesPoller implements Resettable {
return failed;
}
+ /**
+ * Opens an iterator on the stream's first shard, under the same single-shard limit as
+ * {@link #initKinesisIterator}.
+ */
private String initDynamoDbIterator(String streamArn) {
try {
return dynamoDbStreamService.getShardIterator(
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.java
index 82c8ec02d6..319184dd70 100644
--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesService.java
@@ -33,6 +33,9 @@ public class PipesService implements TagHandler, ResourceProvider {
private static final Logger LOG = Logger.getLogger(PipesService.class);
+ private static final int MIN_PARALLELIZATION_FACTOR = 1;
+ private static final int MAX_PARALLELIZATION_FACTOR = 10;
+
private final StorageBackend<String, Pipe> storage;
private final RegionResolver regionResolver;
private final PipesPoller poller;
@@ -324,11 +327,10 @@ public class PipesService implements TagHandler, ResourceProvider {
}
if (source.startsWith("smk://")) {
requireKafkaParameters(sourceParameters, "SelfManagedKafkaParameters");
- return;
- }
- if (source.contains(":kafka:")) {
+ } else if (source.contains(":kafka:")) {
requireKafkaParameters(sourceParameters, "ManagedStreamingKafkaParameters");
}
+ validateParallelizationFactor(source, sourceParameters);
}
private void requireKafkaParameters(JsonNode sourceParameters, String parameterBlock) {
@@ -342,4 +344,49 @@ public class PipesService implements TagHandler, ResourceProvider {
"SourceParameters." + parameterBlock + ".TopicName is required", 400);
}
}
+
+ /**
+ * Applies the AWS bounds on {@code ParallelizationFactor}. It is a member of the Kinesis and
+ * DynamoDB Stream parameter blocks only, so a value carried by a block that does not describe
+ * the pipe's source is rejected rather than silently kept.
+ */
+ private void validateParallelizationFactor(String source, JsonNode sourceParameters) {
+ if (sourceParameters == null) {
+ return;
+ }
+ validateParallelizationFactorBlock(sourceParameters, "KinesisStreamParameters",
+ "Kinesis stream", source.contains(":kinesis:"));
+ validateParallelizationFactorBlock(sourceParameters, "DynamoDBStreamParameters",
+ "DynamoDB Stream", source.contains(":dynamodb:"));
+ }
+
+ private void validateParallelizationFactorBlock(JsonNode sourceParameters, String parameterBlock,
+ String sourceDescription, boolean sourceMatchesBlock) {
+ JsonNode factor = sourceParameters.path(parameterBlock).path("ParallelizationFactor");
+ if (factor.isMissingNode() || factor.isNull()) {
+ return;
+ }
+ String property = "SourceParameters." + parameterBlock + ".ParallelizationFactor";
+ if (!sourceMatchesBlock) {
+ throw new AwsException("ValidationException",
+ property + " is only supported for " + sourceDescription + " sources", 400);
+ }
+ if (!factor.isNumber()) {
+ throw new AwsException("ValidationException",
+ property + " must be a numeric value", 400);
+ }
+ if (!factor.isIntegralNumber()) {
+ throw new AwsException("ValidationException",
+ property + " must be an integer", 400);
+ }
+ // canConvertToInt keeps a value wider than an int out of the bounds check: asLong would
+ // hand back the low 64 bits of a BigInteger, so 2^64 + 5 would read as 5 and be accepted.
+ if (!factor.canConvertToInt()
+ || factor.intValue() < MIN_PARALLELIZATION_FACTOR
+ || factor.intValue() > MAX_PARALLELIZATION_FACTOR) {
+ throw new AwsException("ValidationException",
+ property + " must be between " + MIN_PARALLELIZATION_FACTOR + " and "
+ + MAX_PARALLELIZATION_FACTOR + " (got " + factor.asText() + ")", 400);
+ }
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesIntegrationTest.java
index a8a80a57b2..9e72655869 100644
--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesIntegrationTest.java
@@ -252,4 +252,89 @@ class PipesIntegrationTest {
.then()
.statusCode(400);
}
+
+ @Test
+ @Order(14)
+ void createPipeReturnsParallelizationFactorInResponse() {
+ given()
+ .contentType("application/json")
+ .body("""
+ {
+ "Source": "arn:aws:kinesis:us-east-1:000000000000:stream/pf-stream",
+ "Target": "arn:aws:sqs:us-east-1:000000000000:pf-target",
+ "RoleArn": "arn:aws:iam::000000000000:role/pipe-role",
+ "DesiredState": "STOPPED",
+ "SourceParameters": {
+ "KinesisStreamParameters": {
+ "StartingPosition": "TRIM_HORIZON",
+ "ParallelizationFactor": 4
+ }
+ }
+ }
+ """)
+ .when()
+ .post("/v1/pipes/pf-pipe")
+ .then()
+ .statusCode(200)
+ .body("SourceParameters.KinesisStreamParameters.ParallelizationFactor", equalTo(4));
+
+ given()
+ .contentType("application/json")
+ .when()
+ .get("/v1/pipes/pf-pipe")
+ .then()
+ .statusCode(200)
+ .body("SourceParameters.KinesisStreamParameters.ParallelizationFactor", equalTo(4));
+
+ given()
+ .contentType("application/json")
+ .when()
+ .delete("/v1/pipes/pf-pipe")
+ .then()
+ .statusCode(200);
+ }
+
+ @Test
+ @Order(15)
+ void createPipeParallelizationFactorOutOfRangeReturns400() {
+ given()
+ .contentType("application/json")
+ .body("""
+ {
+ "Source": "arn:aws:kinesis:us-east-1:000000000000:stream/pf-stream",
+ "Target": "arn:aws:sqs:us-east-1:000000000000:pf-target",
+ "RoleArn": "arn:aws:iam::000000000000:role/pipe-role",
+ "DesiredState": "STOPPED",
+ "SourceParameters": {
+ "KinesisStreamParameters": {"ParallelizationFactor": 11}
+ }
+ }
+ """)
+ .when()
+ .post("/v1/pipes/pf-invalid-pipe")
+ .then()
+ .statusCode(400);
+ }
+
+ @Test
+ @Order(16)
+ void createPipeParallelizationFactorOnSqsSourceReturns400() {
+ given()
+ .contentType("application/json")
+ .body("""
+ {
+ "Source": "arn:aws:sqs:us-east-1:000000000000:pf-source",
+ "Target": "arn:aws:sqs:us-east-1:000000000000:pf-target",
+ "RoleArn": "arn:aws:iam::000000000000:role/pipe-role",
+ "DesiredState": "STOPPED",
+ "SourceParameters": {
+ "KinesisStreamParameters": {"ParallelizationFactor": 4}
+ }
+ }
+ """)
+ .when()
+ .post("/v1/pipes/pf-wrong-source-pipe")
+ .then()
+ .statusCode(400);
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.java
index 669e5687b0..e47fc9303d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesServiceTest.java
@@ -1,5 +1,7 @@
package io.github.hectorvent.floci.services.pipes;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
@@ -21,6 +23,8 @@ import static org.mockito.Mockito.when;
class PipesServiceTest {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
private PipesService pipesService;
@BeforeEach
@@ -355,4 +359,159 @@ class PipesServiceTest {
pipesService.describePipe("region-pipe", "eu-west-1"));
assertEquals("NotFoundException", ex.getErrorCode());
}
+
+ @Test
+ void createPipeAcceptsParallelizationFactorOnKinesisSource() {
+ Pipe pipe = pipesService.createPipe("kinesis-pf-pipe",
+ "arn:aws:kinesis:us-east-1:000000000000:stream/events",
+ "arn:aws:sqs:us-east-1:000000000000:target",
+ "arn:aws:iam::000000000000:role/role",
+ null, null, null,
+ sourceParameters("""
+ {"KinesisStreamParameters":{"StartingPosition":"TRIM_HORIZON","ParallelizationFactor":4}}"""),
+ null, null, null, "us-east-1");
+
+ assertEquals(4, pipe.getSourceParameters()
+ .path("KinesisStreamParameters").path("ParallelizationFactor").asInt());
+ }
+
+ @Test
+ void createPipeAcceptsParallelizationFactorOnDynamoDbStreamSource() {
+ Pipe pipe = pipesService.createPipe("ddb-pf-pipe",
+ "arn:aws:dynamodb:us-east-1:000000000000:table/orders/stream/2024-01-01T00:00:00.000",
+ "arn:aws:sqs:us-east-1:000000000000:target",
+ "arn:aws:iam::000000000000:role/role",
+ null, null, null,
+ sourceParameters("""
+ {"DynamoDBStreamParameters":{"StartingPosition":"LATEST","ParallelizationFactor":10}}"""),
+ null, null, null, "us-east-1");
+
+ assertEquals(10, pipe.getSourceParameters()
+ .path("DynamoDBStreamParameters").path("ParallelizationFactor").asInt());
+ }
+
+ @Test
+ void createPipeRejectsParallelizationFactorAboveMaximum() {
+ AwsException ex = assertThrows(AwsException.class, () ->
+ pipesService.createPipe("pf-too-high",
+ "arn:aws:kinesis:us-east-1:000000000000:stream/events",
+ "arn:aws:sqs:us-east-1:000000000000:target",
+ "arn:aws:iam::000000000000:role/role",
+ null, null, null,
+ sourceParameters("""
+ {"KinesisStreamParameters":{"ParallelizationFactor":11}}"""),
+ null, null, null, "us-east-1"));
+
+ assertEquals("ValidationException", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("must be between 1 and 10"), ex.getMessage());
+ }
+
+ @Test
+ void createPipeRejectsParallelizationFactorBelowMinimum() {
+ AwsException ex = assertThrows(AwsException.class, () ->
+ pipesService.createPipe("pf-too-low",
+ "arn:aws:kinesis:us-east-1:000000000000:stream/events",
+ "arn:aws:sqs:us-east-1:000000000000:target",
+ "arn:aws:iam::000000000000:role/role",
+ null, null, null,
+ sourceParameters("""
+ {"KinesisStreamParameters":{"ParallelizationFactor":0}}"""),
+ null, null, null, "us-east-1"));
+
+ assertEquals("ValidationException", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("must be between 1 and 10"), ex.getMessage());
+ }
+
+ @Test
+ void createPipeRejectsFractionalParallelizationFactor() {
+ AwsException ex = assertThrows(AwsException.class, () ->
+ pipesService.createPipe("pf-fractional",
+ "arn:aws:kinesis:us-east-1:000000000000:stream/events",
+ "arn:aws:sqs:us-east-1:000000000000:target",
+ "arn:aws:iam::000000000000:role/role",
+ null, null, null,
+ sourceParameters("""
+ {"KinesisStreamParameters":{"ParallelizationFactor":2.5}}"""),
+ null, null, null, "us-east-1"));
+
+ assertEquals("ValidationException", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("must be an integer"), ex.getMessage());
+ }
+
+ @Test
+ void createPipeRejectsParallelizationFactorWiderThanAnInt() {
+ AwsException ex = assertThrows(AwsException.class, () ->
+ pipesService.createPipe("pf-oversized",
+ "arn:aws:kinesis:us-east-1:000000000000:stream/events",
+ "arn:aws:sqs:us-east-1:000000000000:target",
+ "arn:aws:iam::000000000000:role/role",
+ null, null, null,
+ sourceParameters("""
+ {"KinesisStreamParameters":{"ParallelizationFactor":18446744073709551621}}"""),
+ null, null, null, "us-east-1"));
+
+ assertEquals("ValidationException", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("18446744073709551621"), ex.getMessage());
+ }
+
+ @Test
+ void createPipeRejectsNonNumericParallelizationFactor() {
+ AwsException ex = assertThrows(AwsException.class, () ->
+ pipesService.createPipe("pf-non-numeric",
+ "arn:aws:kinesis:us-east-1:000000000000:stream/events",
+ "arn:aws:sqs:us-east-1:000000000000:target",
+ "arn:aws:iam::000000000000:role/role",
+ null, null, null,
+ sourceParameters("""
+ {"KinesisStreamParameters":{"ParallelizationFactor":"4"}}"""),
+ null, null, null, "us-east-1"));
+
+ assertEquals("ValidationException", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("must be a numeric value"), ex.getMessage());
+ }
+
+ @Test
+ void createPipeRejectsParallelizationFactorOnSqsSource() {
+ AwsException ex = assertThrows(AwsException.class, () ->
+ pipesService.createPipe("pf-wrong-source",
+ "arn:aws:sqs:us-east-1:000000000000:source",
+ "arn:aws:sqs:us-east-1:000000000000:target",
+ "arn:aws:iam::000000000000:role/role",
+ null, null, null,
+ sourceParameters("""
+ {"KinesisStreamParameters":{"ParallelizationFactor":4}}"""),
+ null, null, null, "us-east-1"));
+
+ assertEquals("ValidationException", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("only supported for Kinesis stream sources"), ex.getMessage());
+ }
+
+ @Test
+ void updatePipeRejectsInvalidParallelizationFactor() {
+ pipesService.createPipe("pf-update-pipe",
+ "arn:aws:kinesis:us-east-1:000000000000:stream/events",
+ "arn:aws:sqs:us-east-1:000000000000:target",
+ "arn:aws:iam::000000000000:role/role",
+ null, null, null,
+ sourceParameters("""
+ {"KinesisStreamParameters":{"ParallelizationFactor":2}}"""),
+ null, null, null, "us-east-1");
+
+ AwsException ex = assertThrows(AwsException.class, () ->
+ pipesService.updatePipe("pf-update-pipe", null, null, null, null, null,
+ sourceParameters("""
+ {"KinesisStreamParameters":{"ParallelizationFactor":99}}"""),
+ null, null, "us-east-1"));
+
+ assertEquals("ValidationException", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("must be between 1 and 10"), ex.getMessage());
+ }
+
+ private static JsonNode sourceParameters(String json) {
+ try {
+ return MAPPER.readTree(json);
+ } catch (Exception e) {
+ throw new IllegalStateException("invalid test JSON: " + json, e);
+ }
+ }
}