Light, fluffy, and always free - The AWS Local Emulator alternative
feat(kinesis): add local inspection endpoints for UI (#1880)
themark007486be8f5b07e
diff --git a/docs/services/kinesis.md b/docs/services/kinesis.mdindex f1a0030507..8bca092842 100644--- a/docs/services/kinesis.md+++ b/docs/services/kinesis.md@@ -37,6 +37,18 @@| `UpdateStreamMode` | - |<!-- floci:actions:end -->+## Local Inspection Endpoints++These endpoints are Floci-local read-only helpers for the UI and tests. AWS SDK+traffic should continue to use the JSON 1.1 Kinesis API on `/`.++| Endpoint | Description |+|---|---|+| `GET /_aws/kinesis/streams` | List streams in the resolved region with shard, mode, tag, and record counts |+| `GET /_aws/kinesis/records?StreamName=<name>` | Peek up to 100 stream records with shard attribution without consuming them |+| `GET /_aws/kinesis/records?StreamName=<name>&Limit=<n>` | Peek up to `n` records, capped at 1000 |+| `GET /_aws/kinesis/records?StreamName=<name>&ShardId=<id>` | Peek records for one shard |+## Stream AddressingMost actions accept either `StreamName` or `StreamARN` to identify a stream. When both are provided, `StreamName` takes precedence. `CreateStream` only accepts `StreamName`.@@ -117,4 +129,4 @@ ITERATOR=$(aws kinesis get-shard-iterator \aws kinesis get-records \--shard-iterator $ITERATOR \--endpoint-url $AWS_ENDPOINT_URL-```\ No newline at end of file+```diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionController.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionController.javanew file mode 100644index 0000000000..2c6c1983e0--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionController.java@@ -0,0 +1,163 @@+package io.github.hectorvent.floci.services.kinesis;++import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;+import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;+import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;+import jakarta.inject.Inject;+import jakarta.ws.rs.GET;+import jakarta.ws.rs.Path;+import jakarta.ws.rs.Produces;+import jakarta.ws.rs.QueryParam;+import jakarta.ws.rs.core.Context;+import jakarta.ws.rs.core.HttpHeaders;+import jakarta.ws.rs.core.MediaType;+import jakarta.ws.rs.core.Response;++import java.util.Base64;+import java.util.List;+import java.util.Map;+import java.util.Set;++/**+ * Local inspection endpoint for the Floci UI and test helpers.+ *+ * <p>The AWS-compatible Kinesis API remains the JSON 1.1 endpoint on {@code /};+ * these {@code /_aws} paths are read-only convenience views over local emulator+ * state and do not follow AWS error-response conventions.+ */+@Path("/_aws/kinesis")+@Produces(MediaType.APPLICATION_JSON)+public class KinesisInspectionController {++ private final KinesisService kinesisService;+ private final ObjectMapper objectMapper;+ private final RegionResolver regionResolver;++ @Inject+ public KinesisInspectionController(KinesisService kinesisService,+ ObjectMapper objectMapper,+ RegionResolver regionResolver) {+ this.kinesisService = kinesisService;+ this.objectMapper = objectMapper;+ this.regionResolver = regionResolver;+ }++ @GET+ @Path("/streams")+ public Response getStreams(@Context HttpHeaders headers) {+ String region = regionResolver.resolveRegion(headers);+ ArrayNode streams = objectMapper.createArrayNode();+ for (KinesisStream stream : kinesisService.listStreamDetails(region)) {+ streams.add(streamNode(stream));+ }+ ObjectNode result = objectMapper.createObjectNode();+ result.set("streams", streams);+ return Response.ok(result).build();+ }++ @GET+ @Path("/records")+ public Response getRecords(@Context HttpHeaders headers,+ @QueryParam("StreamName") String streamName,+ @QueryParam("ShardId") String shardId,+ @QueryParam("Limit") Integer limit) {+ if (streamName == null || streamName.isBlank()) {+ return Response.status(400)+ .entity(objectMapper.createObjectNode().put("message", "StreamName query parameter is required"))+ .build();+ }+ if (limit != null && limit <= 0) {+ return Response.status(400)+ .entity(objectMapper.createObjectNode().put("message", "Limit must be greater than 0"))+ .build();+ }++ String region = regionResolver.resolveRegion(headers);+ List<KinesisService.PeekedRecord> captured = kinesisService.peekRecords(streamName, shardId, limit, region);+ ArrayNode records = objectMapper.createArrayNode();+ for (KinesisService.PeekedRecord record : captured) {+ records.add(recordNode(record.shardId(), record.record()));+ }+ ObjectNode result = objectMapper.createObjectNode();+ result.set("records", records);+ return Response.ok(result).build();+ }++ private ObjectNode streamNode(KinesisStream stream) {+ ObjectNode node = objectMapper.createObjectNode();+ node.put("StreamName", stream.getStreamName());+ node.put("StreamARN", stream.getStreamArn());+ node.put("StreamStatus", stream.getStreamStatus());+ node.put("StreamMode", stream.getStreamMode());+ node.put("RetentionPeriodHours", stream.getRetentionPeriodHours());+ node.put("EncryptionType", stream.getEncryptionType());+ if (stream.getKeyId() != null) {+ node.put("KeyId", stream.getKeyId());+ } else {+ node.putNull("KeyId");+ }+ if (stream.getStreamCreationTimestamp() != null) {+ node.put("StreamCreationTimestamp", stream.getStreamCreationTimestamp().toString());+ }+ node.put("OpenShardCount", stream.getShards().stream().filter(s -> !s.isClosed()).count());+ node.put("RecordCount", stream.getShards().stream().mapToInt(s -> s.getRecords().size()).sum());+ ArrayNode enhancedMonitoring = node.putArray("EnhancedMonitoring");+ Set<String> metrics = stream.getEnhancedMonitoringMetrics();+ if (metrics != null) {+ metrics.stream().sorted().forEach(enhancedMonitoring::add);+ }++ ArrayNode shards = node.putArray("Shards");+ for (KinesisShard shard : stream.getShards()) {+ shards.add(shardNode(shard));+ }++ ObjectNode tags = node.putObject("Tags");+ Map<String, String> streamTags = stream.getTags();+ if (streamTags != null) {+ for (Map.Entry<String, String> entry : streamTags.entrySet()) {+ tags.put(entry.getKey(), entry.getValue());+ }+ }+ return node;+ }++ private ObjectNode shardNode(KinesisShard shard) {+ ObjectNode node = objectMapper.createObjectNode();+ node.put("ShardId", shard.getShardId());+ node.put("Closed", shard.isClosed());+ node.put("RecordCount", shard.getRecords().size());+ if (shard.getParentShardId() != null) {+ node.put("ParentShardId", shard.getParentShardId());+ }+ if (shard.getAdjacentParentShardId() != null) {+ node.put("AdjacentParentShardId", shard.getAdjacentParentShardId());+ }+ ObjectNode hashKeyRange = node.putObject("HashKeyRange");+ hashKeyRange.put("StartingHashKey", shard.getHashKeyRange().startingHashKey());+ hashKeyRange.put("EndingHashKey", shard.getHashKeyRange().endingHashKey());+ ObjectNode sequenceNumberRange = node.putObject("SequenceNumberRange");+ sequenceNumberRange.put("StartingSequenceNumber", shard.getSequenceNumberRange().startingSequenceNumber());+ if (shard.getSequenceNumberRange().endingSequenceNumber() != null) {+ sequenceNumberRange.put("EndingSequenceNumber", shard.getSequenceNumberRange().endingSequenceNumber());+ }+ return node;+ }++ private ObjectNode recordNode(String shardId, KinesisRecord record) {+ ObjectNode node = objectMapper.createObjectNode();+ node.put("ShardId", shardId);+ node.put("SequenceNumber", record.getSequenceNumber());+ node.put("PartitionKey", record.getPartitionKey());+ byte[] data = record.getData();+ node.put("Data", data == null ? "" : Base64.getEncoder().encodeToString(data));+ if (record.getApproximateArrivalTimestamp() != null) {+ node.put("ApproximateArrivalTimestamp", record.getApproximateArrivalTimestamp().toString());+ }+ return node;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.javaindex 68097da484..96746e2a43 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java@@ -28,6 +28,8 @@ public class KinesisService {"IteratorAgeMilliseconds", "ALL");private static final Set<String> VALID_STREAM_MODES = Set.of("PROVISIONED", "ON_DEMAND");private static final String DEFAULT_STREAM_MODE = "PROVISIONED";+ private static final int DEFAULT_INSPECTION_RECORD_LIMIT = 100;+ private static final int MAX_INSPECTION_RECORD_LIMIT = 1000;private final StorageBackend<String, KinesisStream> store;private final StorageBackend<String, KinesisConsumer> consumerStore;@@ -112,6 +114,13 @@ public class KinesisService {.toList();}+ public List<KinesisStream> listStreamDetails(String region) {+ String prefix = region + "::";+ return store.scan(key -> key.startsWith(prefix)).stream()+ .sorted(Comparator.comparing(KinesisStream::getStreamName))+ .toList();+ }+public KinesisStream describeStream(String streamName, String region) {return resolveStream(streamName, region);}@@ -511,6 +520,61 @@ public class KinesisService {return response;}+ public record PeekedRecord(String shardId, KinesisRecord record) {}++ /**+ * Returns up to {@code limit} of the most-recent records across all (or one) shard.+ * Records are returned in ascending arrival-timestamp order (oldest first).+ *+ * <p>With no pagination cursor, {@value MAX_INSPECTION_RECORD_LIMIT} is a hard ceiling+ * on the total number of records reachable in a single call regardless of the {@code limit}+ * parameter.+ */+ public List<PeekedRecord> peekRecords(String streamName, String shardId, Integer limit, String region) {+ KinesisStream stream = resolveStream(streamName, region);+ int resolvedLimit = resolveInspectionRecordLimit(limit);+ if (resolvedLimit == 0) {+ return List.of();+ }++ if (shardId != null && !shardId.isBlank()) {+ boolean exists = stream.getShards().stream().anyMatch(shard -> shard.getShardId().equals(shardId));+ if (!exists) {+ throw new AwsException("ResourceNotFoundException", "Shard " + shardId + " not found", 400);+ }+ }++ Comparator<PeekedRecord> oldestFirst = Comparator.comparing(+ peeked -> peeked.record().getApproximateArrivalTimestamp(),+ Comparator.nullsFirst(Comparator.naturalOrder()));+ PriorityQueue<PeekedRecord> newest = new PriorityQueue<>(oldestFirst);+ stream.getShards().stream()+ .filter(shard -> shardId == null || shardId.isBlank() || shard.getShardId().equals(shardId))+ .forEach(shard -> shard.getRecords().forEach(record -> {+ newest.add(new PeekedRecord(shard.getShardId(), record));+ if (newest.size() > resolvedLimit) {+ newest.poll();+ }+ }));++ return newest.stream()+ .sorted(Comparator.comparing(peeked -> peeked.record().getApproximateArrivalTimestamp(),+ Comparator.nullsLast(Comparator.naturalOrder())))+ .toList();+ }++ /**+ * Resolves the caller-supplied limit to a value in {@code [0, MAX_INSPECTION_RECORD_LIMIT]}.+ * Both ends are clamped silently, consistent with the behaviour of+ * {@link #getRecords} and {@link #getRecordsForAccount}.+ */+ private int resolveInspectionRecordLimit(Integer limit) {+ if (limit == null) {+ return DEFAULT_INSPECTION_RECORD_LIMIT;+ }+ return Math.max(0, Math.min(limit, MAX_INSPECTION_RECORD_LIMIT));+ }+/*** Time delta in ms between the last record returned and the shard tip.* Zero when caught up, the shard is empty, or no records were returned.diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionControllerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionControllerIntegrationTest.javanew file mode 100644index 0000000000..7768736453--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionControllerIntegrationTest.java@@ -0,0 +1,141 @@+package io.github.hectorvent.floci.services.kinesis;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.hasItem;+import static org.hamcrest.Matchers.hasSize;++@QuarkusTest+class KinesisInspectionControllerIntegrationTest {++ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void streamsEndpointReturnsKinesisStreamsForUiInspection() {+ String streamName = "ui-inspection-stream";+ createStream(streamName, 2);++ given()+ .when().get("/_aws/kinesis/streams")+ .then()+ .statusCode(200)+ .body("streams.StreamName", hasItem(streamName))+ .body("streams.find { it.StreamName == '" + streamName + "' }.StreamStatus", equalTo("ACTIVE"))+ .body("streams.find { it.StreamName == '" + streamName + "' }.OpenShardCount", equalTo(2))+ .body("streams.find { it.StreamName == '" + streamName + "' }.Shards", hasSize(2));+ }++ @Test+ void recordsEndpointReturnsRecordsWithoutConsumingThem() {+ String streamName = "ui-inspection-records";+ createStream(streamName, 1);++ putRecord(streamName, "aGVsbG8=", "pk");++ given()+ .queryParam("StreamName", streamName)+ .when().get("/_aws/kinesis/records")+ .then()+ .statusCode(200)+ .body("records", hasSize(1))+ .body("records[0].ShardId", equalTo("shardId-000000000000"))+ .body("records[0].PartitionKey", equalTo("pk"))+ .body("records[0].Data", equalTo("aGVsbG8="));++ given()+ .queryParam("StreamName", streamName)+ .when().get("/_aws/kinesis/records")+ .then()+ .statusCode(200)+ .body("records", hasSize(1));+ }++ @Test+ void recordsEndpointHonorsLimit() {+ String streamName = "ui-inspection-record-limit";+ createStream(streamName, 1);+ putRecord(streamName, "b25l", "pk-1"); // oldest+ putRecord(streamName, "dHdv", "pk-2"); // middle+ putRecord(streamName, "dGhyZWU=", "pk-3"); // newest++ // Limit=2 should return the two NEWEST records (pk-2, pk-3) in+ // ascending arrival-timestamp order, not the two oldest.+ given()+ .queryParam("StreamName", streamName)+ .queryParam("Limit", 2)+ .when().get("/_aws/kinesis/records")+ .then()+ .statusCode(200)+ .body("records", hasSize(2))+ .body("records[0].PartitionKey", equalTo("pk-2"))+ .body("records[1].PartitionKey", equalTo("pk-3"));+ }++ @Test+ void recordsEndpointRejectsNonPositiveLimit() {+ String streamName = "ui-inspection-record-limit-zero";+ createStream(streamName, 1);++ given()+ .queryParam("StreamName", streamName)+ .queryParam("Limit", 0)+ .when().get("/_aws/kinesis/records")+ .then()+ .statusCode(400)+ .body("message", equalTo("Limit must be greater than 0"));+ }++ @Test+ void recordsEndpointRejectsUnknownShard() {+ String streamName = "ui-inspection-unknown-shard";+ createStream(streamName, 1);++ given()+ .queryParam("StreamName", streamName)+ .queryParam("ShardId", "shardId-999999999999")+ .when().get("/_aws/kinesis/records")+ .then()+ .statusCode(400);+ }++ @Test+ void recordsEndpointRequiresStreamName() {+ given()+ .when().get("/_aws/kinesis/records")+ .then()+ .statusCode(400)+ .body("message", equalTo("StreamName query parameter is required"));+ }++ private static void createStream(String streamName, int shardCount) {+ given()+ .header("X-Amz-Target", "Kinesis_20131202.CreateStream")+ .contentType(CONTENT_TYPE)+ .body("""+ {"StreamName": "%s", "ShardCount": %d}+ """.formatted(streamName, shardCount))+ .when().post("/")+ .then().statusCode(200);+ }++ private static void putRecord(String streamName, String data, String partitionKey) {+ given()+ .header("X-Amz-Target", "Kinesis_20131202.PutRecord")+ .contentType(CONTENT_TYPE)+ .body("""+ {"StreamName": "%s", "Data": "%s", "PartitionKey": "%s"}+ """.formatted(streamName, data, partitionKey))+ .when().post("/")+ .then().statusCode(200);+ }+}