Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(kinesis): add local inspection endpoints for UI (#1880)

themark007486be8f5b07e
diff --git a/docs/services/kinesis.md b/docs/services/kinesis.md
index f1a0030507..8bca092842 100644
--- a/docs/services/kinesis.md
+++ b/docs/services/kinesis.md
@@ -37,6 +37,18 @@
| `UpdateStreamMode` | - |
<!-- floci:actions:end -->
+## Local Inspection Endpoints
+
+These endpoints are Floci-local read-only helpers for the UI and tests. AWS SDK
+traffic should continue to use the JSON 1.1 Kinesis API on `/`.
+
+| Endpoint | Description |
+|---|---|
+| `GET /_aws/kinesis/streams` | List streams in the resolved region with shard, mode, tag, and record counts |
+| `GET /_aws/kinesis/records?StreamName=<name>` | Peek up to 100 stream records with shard attribution without consuming them |
+| `GET /_aws/kinesis/records?StreamName=<name>&Limit=<n>` | Peek up to `n` records, capped at 1000 |
+| `GET /_aws/kinesis/records?StreamName=<name>&ShardId=<id>` | Peek records for one shard |
+
## Stream Addressing
Most actions accept either `StreamName` or `StreamARN` to identify a stream. When both are provided, `StreamName` takes precedence. `CreateStream` only accepts `StreamName`.
@@ -117,4 +129,4 @@ ITERATOR=$(aws kinesis get-shard-iterator \
aws kinesis get-records \
--shard-iterator $ITERATOR \
--endpoint-url $AWS_ENDPOINT_URL
-```
\ No newline at end of file
+```
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionController.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionController.java
new file mode 100644
index 0000000000..2c6c1983e0
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionController.java
@@ -0,0 +1,163 @@
+package io.github.hectorvent.floci.services.kinesis;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;
+import jakarta.inject.Inject;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.QueryParam;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.HttpHeaders;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Local inspection endpoint for the Floci UI and test helpers.
+ *
+ * <p>The AWS-compatible Kinesis API remains the JSON 1.1 endpoint on {@code /};
+ * these {@code /_aws} paths are read-only convenience views over local emulator
+ * state and do not follow AWS error-response conventions.
+ */
+@Path("/_aws/kinesis")
+@Produces(MediaType.APPLICATION_JSON)
+public class KinesisInspectionController {
+
+ private final KinesisService kinesisService;
+ private final ObjectMapper objectMapper;
+ private final RegionResolver regionResolver;
+
+ @Inject
+ public KinesisInspectionController(KinesisService kinesisService,
+ ObjectMapper objectMapper,
+ RegionResolver regionResolver) {
+ this.kinesisService = kinesisService;
+ this.objectMapper = objectMapper;
+ this.regionResolver = regionResolver;
+ }
+
+ @GET
+ @Path("/streams")
+ public Response getStreams(@Context HttpHeaders headers) {
+ String region = regionResolver.resolveRegion(headers);
+ ArrayNode streams = objectMapper.createArrayNode();
+ for (KinesisStream stream : kinesisService.listStreamDetails(region)) {
+ streams.add(streamNode(stream));
+ }
+ ObjectNode result = objectMapper.createObjectNode();
+ result.set("streams", streams);
+ return Response.ok(result).build();
+ }
+
+ @GET
+ @Path("/records")
+ public Response getRecords(@Context HttpHeaders headers,
+ @QueryParam("StreamName") String streamName,
+ @QueryParam("ShardId") String shardId,
+ @QueryParam("Limit") Integer limit) {
+ if (streamName == null || streamName.isBlank()) {
+ return Response.status(400)
+ .entity(objectMapper.createObjectNode().put("message", "StreamName query parameter is required"))
+ .build();
+ }
+ if (limit != null && limit <= 0) {
+ return Response.status(400)
+ .entity(objectMapper.createObjectNode().put("message", "Limit must be greater than 0"))
+ .build();
+ }
+
+ String region = regionResolver.resolveRegion(headers);
+ List<KinesisService.PeekedRecord> captured = kinesisService.peekRecords(streamName, shardId, limit, region);
+ ArrayNode records = objectMapper.createArrayNode();
+ for (KinesisService.PeekedRecord record : captured) {
+ records.add(recordNode(record.shardId(), record.record()));
+ }
+ ObjectNode result = objectMapper.createObjectNode();
+ result.set("records", records);
+ return Response.ok(result).build();
+ }
+
+ private ObjectNode streamNode(KinesisStream stream) {
+ ObjectNode node = objectMapper.createObjectNode();
+ node.put("StreamName", stream.getStreamName());
+ node.put("StreamARN", stream.getStreamArn());
+ node.put("StreamStatus", stream.getStreamStatus());
+ node.put("StreamMode", stream.getStreamMode());
+ node.put("RetentionPeriodHours", stream.getRetentionPeriodHours());
+ node.put("EncryptionType", stream.getEncryptionType());
+ if (stream.getKeyId() != null) {
+ node.put("KeyId", stream.getKeyId());
+ } else {
+ node.putNull("KeyId");
+ }
+ if (stream.getStreamCreationTimestamp() != null) {
+ node.put("StreamCreationTimestamp", stream.getStreamCreationTimestamp().toString());
+ }
+ node.put("OpenShardCount", stream.getShards().stream().filter(s -> !s.isClosed()).count());
+ node.put("RecordCount", stream.getShards().stream().mapToInt(s -> s.getRecords().size()).sum());
+ ArrayNode enhancedMonitoring = node.putArray("EnhancedMonitoring");
+ Set<String> metrics = stream.getEnhancedMonitoringMetrics();
+ if (metrics != null) {
+ metrics.stream().sorted().forEach(enhancedMonitoring::add);
+ }
+
+ ArrayNode shards = node.putArray("Shards");
+ for (KinesisShard shard : stream.getShards()) {
+ shards.add(shardNode(shard));
+ }
+
+ ObjectNode tags = node.putObject("Tags");
+ Map<String, String> streamTags = stream.getTags();
+ if (streamTags != null) {
+ for (Map.Entry<String, String> entry : streamTags.entrySet()) {
+ tags.put(entry.getKey(), entry.getValue());
+ }
+ }
+ return node;
+ }
+
+ private ObjectNode shardNode(KinesisShard shard) {
+ ObjectNode node = objectMapper.createObjectNode();
+ node.put("ShardId", shard.getShardId());
+ node.put("Closed", shard.isClosed());
+ node.put("RecordCount", shard.getRecords().size());
+ if (shard.getParentShardId() != null) {
+ node.put("ParentShardId", shard.getParentShardId());
+ }
+ if (shard.getAdjacentParentShardId() != null) {
+ node.put("AdjacentParentShardId", shard.getAdjacentParentShardId());
+ }
+ ObjectNode hashKeyRange = node.putObject("HashKeyRange");
+ hashKeyRange.put("StartingHashKey", shard.getHashKeyRange().startingHashKey());
+ hashKeyRange.put("EndingHashKey", shard.getHashKeyRange().endingHashKey());
+ ObjectNode sequenceNumberRange = node.putObject("SequenceNumberRange");
+ sequenceNumberRange.put("StartingSequenceNumber", shard.getSequenceNumberRange().startingSequenceNumber());
+ if (shard.getSequenceNumberRange().endingSequenceNumber() != null) {
+ sequenceNumberRange.put("EndingSequenceNumber", shard.getSequenceNumberRange().endingSequenceNumber());
+ }
+ return node;
+ }
+
+ private ObjectNode recordNode(String shardId, KinesisRecord record) {
+ ObjectNode node = objectMapper.createObjectNode();
+ node.put("ShardId", shardId);
+ node.put("SequenceNumber", record.getSequenceNumber());
+ node.put("PartitionKey", record.getPartitionKey());
+ byte[] data = record.getData();
+ node.put("Data", data == null ? "" : Base64.getEncoder().encodeToString(data));
+ if (record.getApproximateArrivalTimestamp() != null) {
+ node.put("ApproximateArrivalTimestamp", record.getApproximateArrivalTimestamp().toString());
+ }
+ return node;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
index 68097da484..96746e2a43 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
@@ -28,6 +28,8 @@ public class KinesisService {
"IteratorAgeMilliseconds", "ALL");
private static final Set<String> VALID_STREAM_MODES = Set.of("PROVISIONED", "ON_DEMAND");
private static final String DEFAULT_STREAM_MODE = "PROVISIONED";
+ private static final int DEFAULT_INSPECTION_RECORD_LIMIT = 100;
+ private static final int MAX_INSPECTION_RECORD_LIMIT = 1000;
private final StorageBackend<String, KinesisStream> store;
private final StorageBackend<String, KinesisConsumer> consumerStore;
@@ -112,6 +114,13 @@ public class KinesisService {
.toList();
}
+ public List<KinesisStream> listStreamDetails(String region) {
+ String prefix = region + "::";
+ return store.scan(key -> key.startsWith(prefix)).stream()
+ .sorted(Comparator.comparing(KinesisStream::getStreamName))
+ .toList();
+ }
+
public KinesisStream describeStream(String streamName, String region) {
return resolveStream(streamName, region);
}
@@ -511,6 +520,61 @@ public class KinesisService {
return response;
}
+ public record PeekedRecord(String shardId, KinesisRecord record) {}
+
+ /**
+ * Returns up to {@code limit} of the most-recent records across all (or one) shard.
+ * Records are returned in ascending arrival-timestamp order (oldest first).
+ *
+ * <p>With no pagination cursor, {@value MAX_INSPECTION_RECORD_LIMIT} is a hard ceiling
+ * on the total number of records reachable in a single call regardless of the {@code limit}
+ * parameter.
+ */
+ public List<PeekedRecord> peekRecords(String streamName, String shardId, Integer limit, String region) {
+ KinesisStream stream = resolveStream(streamName, region);
+ int resolvedLimit = resolveInspectionRecordLimit(limit);
+ if (resolvedLimit == 0) {
+ return List.of();
+ }
+
+ if (shardId != null && !shardId.isBlank()) {
+ boolean exists = stream.getShards().stream().anyMatch(shard -> shard.getShardId().equals(shardId));
+ if (!exists) {
+ throw new AwsException("ResourceNotFoundException", "Shard " + shardId + " not found", 400);
+ }
+ }
+
+ Comparator<PeekedRecord> oldestFirst = Comparator.comparing(
+ peeked -> peeked.record().getApproximateArrivalTimestamp(),
+ Comparator.nullsFirst(Comparator.naturalOrder()));
+ PriorityQueue<PeekedRecord> newest = new PriorityQueue<>(oldestFirst);
+ stream.getShards().stream()
+ .filter(shard -> shardId == null || shardId.isBlank() || shard.getShardId().equals(shardId))
+ .forEach(shard -> shard.getRecords().forEach(record -> {
+ newest.add(new PeekedRecord(shard.getShardId(), record));
+ if (newest.size() > resolvedLimit) {
+ newest.poll();
+ }
+ }));
+
+ return newest.stream()
+ .sorted(Comparator.comparing(peeked -> peeked.record().getApproximateArrivalTimestamp(),
+ Comparator.nullsLast(Comparator.naturalOrder())))
+ .toList();
+ }
+
+ /**
+ * Resolves the caller-supplied limit to a value in {@code [0, MAX_INSPECTION_RECORD_LIMIT]}.
+ * Both ends are clamped silently, consistent with the behaviour of
+ * {@link #getRecords} and {@link #getRecordsForAccount}.
+ */
+ private int resolveInspectionRecordLimit(Integer limit) {
+ if (limit == null) {
+ return DEFAULT_INSPECTION_RECORD_LIMIT;
+ }
+ return Math.max(0, Math.min(limit, MAX_INSPECTION_RECORD_LIMIT));
+ }
+
/**
* Time delta in ms between the last record returned and the shard tip.
* Zero when caught up, the shard is empty, or no records were returned.
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionControllerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionControllerIntegrationTest.java
new file mode 100644
index 0000000000..7768736453
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionControllerIntegrationTest.java
@@ -0,0 +1,141 @@
+package io.github.hectorvent.floci.services.kinesis;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+
+@QuarkusTest
+class KinesisInspectionControllerIntegrationTest {
+
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void streamsEndpointReturnsKinesisStreamsForUiInspection() {
+ String streamName = "ui-inspection-stream";
+ createStream(streamName, 2);
+
+ given()
+ .when().get("/_aws/kinesis/streams")
+ .then()
+ .statusCode(200)
+ .body("streams.StreamName", hasItem(streamName))
+ .body("streams.find { it.StreamName == '" + streamName + "' }.StreamStatus", equalTo("ACTIVE"))
+ .body("streams.find { it.StreamName == '" + streamName + "' }.OpenShardCount", equalTo(2))
+ .body("streams.find { it.StreamName == '" + streamName + "' }.Shards", hasSize(2));
+ }
+
+ @Test
+ void recordsEndpointReturnsRecordsWithoutConsumingThem() {
+ String streamName = "ui-inspection-records";
+ createStream(streamName, 1);
+
+ putRecord(streamName, "aGVsbG8=", "pk");
+
+ given()
+ .queryParam("StreamName", streamName)
+ .when().get("/_aws/kinesis/records")
+ .then()
+ .statusCode(200)
+ .body("records", hasSize(1))
+ .body("records[0].ShardId", equalTo("shardId-000000000000"))
+ .body("records[0].PartitionKey", equalTo("pk"))
+ .body("records[0].Data", equalTo("aGVsbG8="));
+
+ given()
+ .queryParam("StreamName", streamName)
+ .when().get("/_aws/kinesis/records")
+ .then()
+ .statusCode(200)
+ .body("records", hasSize(1));
+ }
+
+ @Test
+ void recordsEndpointHonorsLimit() {
+ String streamName = "ui-inspection-record-limit";
+ createStream(streamName, 1);
+ putRecord(streamName, "b25l", "pk-1"); // oldest
+ putRecord(streamName, "dHdv", "pk-2"); // middle
+ putRecord(streamName, "dGhyZWU=", "pk-3"); // newest
+
+ // Limit=2 should return the two NEWEST records (pk-2, pk-3) in
+ // ascending arrival-timestamp order, not the two oldest.
+ given()
+ .queryParam("StreamName", streamName)
+ .queryParam("Limit", 2)
+ .when().get("/_aws/kinesis/records")
+ .then()
+ .statusCode(200)
+ .body("records", hasSize(2))
+ .body("records[0].PartitionKey", equalTo("pk-2"))
+ .body("records[1].PartitionKey", equalTo("pk-3"));
+ }
+
+ @Test
+ void recordsEndpointRejectsNonPositiveLimit() {
+ String streamName = "ui-inspection-record-limit-zero";
+ createStream(streamName, 1);
+
+ given()
+ .queryParam("StreamName", streamName)
+ .queryParam("Limit", 0)
+ .when().get("/_aws/kinesis/records")
+ .then()
+ .statusCode(400)
+ .body("message", equalTo("Limit must be greater than 0"));
+ }
+
+ @Test
+ void recordsEndpointRejectsUnknownShard() {
+ String streamName = "ui-inspection-unknown-shard";
+ createStream(streamName, 1);
+
+ given()
+ .queryParam("StreamName", streamName)
+ .queryParam("ShardId", "shardId-999999999999")
+ .when().get("/_aws/kinesis/records")
+ .then()
+ .statusCode(400);
+ }
+
+ @Test
+ void recordsEndpointRequiresStreamName() {
+ given()
+ .when().get("/_aws/kinesis/records")
+ .then()
+ .statusCode(400)
+ .body("message", equalTo("StreamName query parameter is required"));
+ }
+
+ private static void createStream(String streamName, int shardCount) {
+ given()
+ .header("X-Amz-Target", "Kinesis_20131202.CreateStream")
+ .contentType(CONTENT_TYPE)
+ .body("""
+ {"StreamName": "%s", "ShardCount": %d}
+ """.formatted(streamName, shardCount))
+ .when().post("/")
+ .then().statusCode(200);
+ }
+
+ private static void putRecord(String streamName, String data, String partitionKey) {
+ given()
+ .header("X-Amz-Target", "Kinesis_20131202.PutRecord")
+ .contentType(CONTENT_TYPE)
+ .body("""
+ {"StreamName": "%s", "Data": "%s", "PartitionKey": "%s"}
+ """.formatted(streamName, data, partitionKey))
+ .when().post("/")
+ .then().statusCode(200);
+ }
+}