Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(firehose): deliver records from a Kinesis stream source (#2528)

Allen Sanborn009ac0b4a62e
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
index 62812ce544..eed58a49c0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
@@ -11,6 +11,10 @@ import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescript
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.KinesisStreamSource;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
import io.github.hectorvent.floci.services.firehose.model.Record;
+import io.github.hectorvent.floci.services.kinesis.KinesisService;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.PutObjectOptions;
import io.quarkus.runtime.ShutdownDelayInitiatedEvent;
@@ -44,12 +48,48 @@ public class FirehoseService implements ResourceProvider {
private static final String DEFAULT_BUCKET = "floci-firehose-results";
private static final int DEFAULT_BUFFERING_INTERVAL_SECONDS = 300;
private static final int DEFAULT_BUFFERING_SIZE_MBS = 5;
+ // Per shard, per tick. GetRecords caps at 1000 anyway; a smaller page keeps one
+ // busy source stream from monopolising the single flusher thread.
+ private static final int SOURCE_POLL_LIMIT = 500;
private final StorageBackend<String, DeliveryStreamDescription> streamStore;
private final Map<String, List<byte[]>> buffers = new ConcurrentHashMap<>();
private final Map<String, Instant> bufferSince = new ConcurrentHashMap<>();
private final S3Service s3Service;
+ private final KinesisService kinesisService;
private final RegionResolver regionResolver;
+ // Per delivery stream, the shard iterator each source shard has been consumed to.
+ // Iterators are the checkpoint: a delivery stream restarted mid-shard resumes from
+ // the tip it reached, and one that has never polled starts at DeliveryStartTimestamp.
+ //
+ // Persisted, unlike buffers/bufferSince above, because losing it is not merely losing
+ // what was in flight: an emulator restart would rebuild every iterator from
+ // DeliveryStartTimestamp and re-deliver the source stream's whole retained history to
+ // S3. A Kinesis shard iterator is a self-describing token
+ // (streamName|shardId|type|sequenceNumber|index|timestamp, base64) with no
+ // JVM-lifetime component, and KinesisService never trims a shard's record list, so a
+ // checkpoint written before a restart still resolves to the same position after one.
+ //
+ // This store holds only COMMITTED positions: a shard is advanced here once the
+ // records read up to that point have actually landed in S3. See
+ // pendingSourceIterators for why the two halves cannot be one map.
+ private final StorageBackend<String, Map<String, String>> sourceIteratorStore;
+ // The advanced-but-not-yet-durable half of the checkpoint, deliberately in memory.
+ //
+ // Polled records do not go straight to S3; they go into buffers above and reach S3
+ // only when a flush trips. Persisting the advanced iterator at poll time would let
+ // the durable checkpoint outrun durable delivery: a crash after the put but before
+ // the flush loses the buffer and resumes past records that never reached S3, which is
+ // silent, permanent loss. So the poller advances here, and only a flush that actually
+ // wrote to S3 promotes these into sourceIteratorStore.
+ //
+ // Losing this map costs nothing but duplicates: the restored poller re-reads records
+ // that were buffered and never delivered, and delivers them. Kinesis Data Firehose is
+ // at-least-once, so a duplicate is correct-if-untidy where a skip is not.
+ //
+ // Values are replaced whole, never mutated in place, so a flusher that snapshots one
+ // commits exactly the position that snapshot covered.
+ private final Map<String, Map<String, String>> pendingSourceIterators = new ConcurrentHashMap<>();
private final Clock clock;
private final long tickIntervalSeconds;
private final int flushRecordCount;
@@ -57,11 +97,14 @@ public class FirehoseService implements ResourceProvider {
private final ScheduledExecutorService flushExecutor;
@Inject
- public FirehoseService(StorageFactory storageFactory, S3Service s3Service, RegionResolver regionResolver,
- Clock clock, EmulatorConfig config) {
+ public FirehoseService(StorageFactory storageFactory, S3Service s3Service, KinesisService kinesisService,
+ RegionResolver regionResolver, Clock clock, EmulatorConfig config) {
this.streamStore = storageFactory.create("firehose", "streams.json",
new TypeReference<Map<String, DeliveryStreamDescription>>() {});
+ this.sourceIteratorStore = storageFactory.create("firehose", "source-iterators.json",
+ new TypeReference<Map<String, Map<String, String>>>() {});
this.s3Service = s3Service;
+ this.kinesisService = kinesisService;
this.regionResolver = regionResolver;
this.clock = clock;
this.tickIntervalSeconds = Math.max(1, config.services().firehose().tickIntervalSeconds());
@@ -85,7 +128,10 @@ public class FirehoseService implements ResourceProvider {
// ShutdownDelayInitiatedEvent fires before every ShutdownEvent observer, so this
// drain lands the pending records in S3 while EmulatorLifecycle.onStop can still
- // persist them to disk via storageFactory.flushAll().
+ // persist them to disk via storageFactory.flushAll(). Each flush also commits the
+ // source checkpoint it just made durable, so a graceful shutdown neither loses
+ // records nor re-delivers them; storageFactory.flushAll() then writes that
+ // checkpoint out with everything else.
void onPreShutdown(@Observes ShutdownDelayInitiatedEvent ignored) {
flushExecutor.shutdownNow();
buffers.keySet().forEach(this::flush);
@@ -93,6 +139,7 @@ public class FirehoseService implements ResourceProvider {
void tickSafely() {
try {
+ pollKinesisSources();
flushDueBuffers(clock.instant());
} catch (Throwable t) {
LOG.warnv("Firehose buffer flush tick failed: {0}", t.getMessage());
@@ -157,6 +204,15 @@ public class FirehoseService implements ResourceProvider {
validateBufferingHints(s3Config);
String arn = AwsArnUtils.Arn.of("firehose", regionResolver.getDefaultRegion(), regionResolver.getAccountId(), "deliverystream/" + name).toString();
+ // CreateDeliveryStream's KinesisStreamSourceConfiguration carries only the ARN and
+ // role -- DeliveryStartTimestamp exists only on the Description shape, which AWS
+ // fills in at creation. Stamping it here, at creation time, is what makes the first
+ // shard iterator in pollKinesisSource start AT_TIMESTAMP=now instead of falling back
+ // to TRIM_HORIZON: attaching Firehose to a stream that already holds records must not
+ // backfill its history into S3.
+ if (source != null && source.getDeliveryStartTimestamp() == null) {
+ source.setDeliveryStartTimestamp(Instant.now());
+ }
DeliveryStreamDescription description = new DeliveryStreamDescription(name, arn, s3Config, source);
description.setAccountId(regionResolver.getAccountId());
description.setTags(tags);
@@ -312,6 +368,8 @@ public class FirehoseService implements ResourceProvider {
// reached the bucket after DeleteDeliveryStream completed.
buffers.remove(name);
bufferSince.remove(name);
+ pendingSourceIterators.remove(name);
+ sourceIteratorStore.delete(name);
LOG.infov("Deleted Firehose delivery stream: {0}", name);
}
@@ -320,22 +378,143 @@ public class FirehoseService implements ResourceProvider {
.map(DeliveryStreamDescription::getDeliveryStreamName).toList();
}
+ /**
+ * Ingests records written to a delivery stream's Kinesis source.
+ *
+ * <p>A {@code KinesisStreamAsSource} delivery stream has no PutRecord of its own on
+ * real AWS: everything it delivers arrives by Firehose reading the source stream.
+ * Without this the source configuration is recorded and then ignored, so a stream
+ * created with one accepts the Terraform apply, reports healthy, and silently never
+ * delivers -- which is how terraform-aws-messaging's TestKinesisFirehose fails
+ * ("no objects found in the bucket" after six retries) while every API call in it
+ * succeeds.
+ *
+ * <p>Consumption uses the ordinary shard-iterator API, so the records seen here are
+ * exactly the records a GetRecords consumer would see, and the iterator is the
+ * checkpoint. The first iterator starts at {@code DeliveryStartTimestamp} rather than
+ * TRIM_HORIZON, matching AWS: attaching Firehose to a stream that already holds
+ * records does not backfill them.
+ */
+ void pollKinesisSources() {
+ for (DeliveryStreamDescription stream : streamStore.scan(k -> true)) {
+ KinesisStreamSource source = stream.getSource() == null
+ ? null : stream.getSource().getKinesisStreamSourceDescription();
+ if (source == null || source.getKinesisStreamArn() == null) {
+ continue;
+ }
+ try {
+ pollKinesisSource(stream.getDeliveryStreamName(), source);
+ } catch (Exception e) {
+ // A source stream that was deleted (or was never created) is the caller's
+ // business, not a reason to stop polling every other delivery stream.
+ LOG.debugv("Firehose source poll failed for stream {0}: {1}",
+ stream.getDeliveryStreamName(), e.getMessage());
+ }
+ }
+ }
+
+ private void pollKinesisSource(String deliveryStreamName, KinesisStreamSource source) {
+ AwsArnUtils.Arn arn = AwsArnUtils.parse(source.getKinesisStreamArn());
+ String sourceName = arn.resource().startsWith("stream/")
+ ? arn.resource().substring("stream/".length())
+ : arn.resource();
+ String region = arn.region() == null || arn.region().isBlank()
+ ? regionResolver.getDefaultRegion() : arn.region();
+ KinesisStream sourceStream = kinesisService.describeStream(sourceName, region);
+ for (KinesisShard shard : sourceStream.getShards()) {
+ // Where to read from: the pending position if a flush still owes these
+ // records to S3, otherwise the committed one. Reading pending is what stops
+ // a second poll re-reading what the first already buffered; committing only
+ // on flush is what stops the durable checkpoint outrunning delivery.
+ String iterator = sourceIteratorsInUse(deliveryStreamName).get(shard.getShardId());
+ if (iterator == null) {
+ Instant start = source.getDeliveryStartTimestamp();
+ iterator = start == null
+ ? kinesisService.getShardIterator(sourceName, shard.getShardId(),
+ "TRIM_HORIZON", null, region)
+ : kinesisService.getShardIterator(sourceName, shard.getShardId(),
+ "AT_TIMESTAMP", null, start.toEpochMilli(), region);
+ }
+ Map<String, Object> page = kinesisService.getRecords(iterator, SOURCE_POLL_LIMIT, region);
+ @SuppressWarnings("unchecked")
+ List<KinesisRecord> records = (List<KinesisRecord>) page.get("Records");
+ String next = (String) page.get("NextShardIterator");
+ String advanced = next != null ? next : iterator;
+ String shardId = shard.getShardId();
+ if (records == null || records.isEmpty()) {
+ // Nothing was buffered, so this advance owes S3 nothing and is safe to
+ // park as pending on its own.
+ advanceSourceIterator(deliveryStreamName, shardId, advanced);
+ continue;
+ }
+ // Through putRecordBatch so source records buffer, and trigger a flush, on
+ // exactly the same terms as records handed to PutRecord directly. The
+ // advance runs under the buffer lock together with the records it covers, so
+ // no flush can ever see one without the other.
+ putRecordBatch(deliveryStreamName, records.stream()
+ .map(record -> new Record(record.getData()))
+ .toList(),
+ () -> advanceSourceIterator(deliveryStreamName, shardId, advanced));
+ }
+ }
+
+ /** The position the poller reads from: pending if a flush still owes it, else committed. */
+ private Map<String, String> sourceIteratorsInUse(String deliveryStreamName) {
+ Map<String, String> pending = pendingSourceIterators.get(deliveryStreamName);
+ return pending != null ? pending : sourceIteratorStore.get(deliveryStreamName).orElseGet(Map::of);
+ }
+
+ // Rebased on whatever is current rather than on a map carried across the shard loop,
+ // so that a flush which discarded pending (because its S3 write failed) rolls the
+ // other shards back to committed instead of having this poll re-assert them.
+ private void advanceSourceIterator(String deliveryStreamName, String shardId, String iterator) {
+ Map<String, String> advanced = new LinkedHashMap<>(sourceIteratorsInUse(deliveryStreamName));
+ advanced.put(shardId, iterator);
+ pendingSourceIterators.put(deliveryStreamName, Collections.unmodifiableMap(advanced));
+ }
+
+ // Merged rather than written whole: two flushes can complete out of order, and a
+ // whole-map write would then drop a shard's newer committed position. Merging can
+ // still move one shard backwards, which costs a duplicate and never a skip.
+ private void commitSourceIterators(String deliveryStreamName, Map<String, String> checkpoint) {
+ if (checkpoint == null || checkpoint.isEmpty()) {
+ return;
+ }
+ // Copied out and written back whole: the store may hand back its own instance,
+ // and a persistent backend only learns of a change through put().
+ Map<String, String> committed =
+ new LinkedHashMap<>(sourceIteratorStore.get(deliveryStreamName).orElseGet(Map::of));
+ committed.putAll(checkpoint);
+ sourceIteratorStore.put(deliveryStreamName, committed);
+ }
+
public void putRecord(String streamName, Record record) {
putRecordBatch(streamName, List.of(record));
}
public void putRecordBatch(String streamName, List<Record> records) {
+ putRecordBatch(streamName, records, null);
+ }
+
+ /**
+ * @param checkpointAdvance run under the buffer lock once the records are buffered,
+ * or null for records that are not read from a source stream.
+ */
+ private void putRecordBatch(String streamName, List<Record> records, Runnable checkpointAdvance) {
DeliveryStreamDescription stream = describeDeliveryStream(streamName);
List<byte[]> buffer = buffers.computeIfAbsent(
streamName, k -> Collections.synchronizedList(new ArrayList<>()));
long bufferedBytes = 0;
int bufferedCount;
- // Records and their buffering-start timestamp move together under the
- // buffer lock so the flusher never sees one without the other.
+ // Records, their buffering-start timestamp and any source checkpoint advance move
+ // together under the buffer lock so the flusher never sees one without the other.
synchronized (buffer) {
for (Record r : records) {
buffer.add(r.getData());
}
+ if (checkpointAdvance != null) {
+ checkpointAdvance.run();
+ }
bufferSince.putIfAbsent(streamName, clock.instant());
bufferedCount = buffer.size();
for (byte[] data : buffer) {
@@ -367,13 +546,20 @@ public class FirehoseService implements ResourceProvider {
}
List<byte[]> toFlush;
+ // Taken with the records, under the same lock: this is the position the records
+ // about to be written cover, and committing it is this flush's job.
+ Map<String, String> checkpoint;
synchronized (buffer) {
toFlush = new ArrayList<>(buffer);
buffer.clear();
bufferSince.remove(streamName);
+ checkpoint = pendingSourceIterators.remove(streamName);
}
if (toFlush.isEmpty()) {
- // Lost the race against a concurrent flush; nothing left to deliver.
+ // Lost the race against a concurrent flush; nothing left to deliver. Any
+ // checkpoint taken here covers no undelivered records -- it can only have
+ // come from a poll that read an empty page -- so it commits as it stands.
+ commitSourceIterators(streamName, checkpoint);
return;
}
@@ -406,8 +592,13 @@ public class FirehoseService implements ResourceProvider {
new PutObjectOptions().withContentEncoding(compression.contentEncoding()));
LOG.infov("Flushed {0} records from stream {1} to s3://{2}/{3} ({4})",
toFlush.size(), streamName, bucket, key, compression.wireValue());
+ // Only now: the records these iterators were read past are durable.
+ commitSourceIterators(streamName, checkpoint);
} catch (Exception e) {
LOG.errorv("Failed to flush Firehose stream {0}: {1}", streamName, e.getMessage());
+ // checkpoint is deliberately neither committed nor put back. The durable
+ // checkpoint stays where it was, so the next poll reads these records again
+ // and this failed delivery repairs itself.
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushIntegrationTest.java
index 267c57fdc6..68b04c61bc 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushIntegrationTest.java
@@ -101,6 +101,102 @@ class FirehoseFlushIntegrationTest {
.body(equalTo("{\"n\":42}\n")));
}
+ /**
+ * Wire-level coverage of the gap in {@code kinesisSourceDoesNotBackfillRecordsFromBeforeDeliveryStart}
+ * (FirehoseServiceTest): that test calls {@code FirehoseService.createDeliveryStream}
+ * directly and hand-sets {@code DeliveryStartTimestamp} on the source, stepping around
+ * the actual API path. CreateDeliveryStream's wire request
+ * (KinesisStreamSourceConfiguration) never carries that field -- AWS fills it in on the
+ * Description shape at creation -- so this drives creation through the real
+ * X-Amz-Target: Firehose_20150804.CreateDeliveryStream call against a Kinesis stream
+ * that already holds a record, and asserts that record is never delivered while one
+ * added afterward is.
+ */
+ @Test
+ void kinesisSourceCreatedThroughTheApiDoesNotBackfillPreExistingRecords() throws InterruptedException {
+ String sourceStream = "flush-kinesis-source-stream";
+ String deliveryStream = "flush-kinesis-source-delivery";
+ String bucket = "flush-kinesis-source-archive";
+ String sourceArn = "arn:aws:kinesis:us-east-1:000000000000:stream/" + sourceStream;
+
+ createKinesisStream(sourceStream);
+ putKinesisRecord(sourceStream, "old");
+
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "DeliveryStreamType": "KinesisStreamAsSource",
+ "KinesisStreamSourceConfiguration": {
+ "KinesisStreamARN": "%s",
+ "RoleARN": "arn:aws:iam::000000000000:role/firehose-source-role"
+ },
+ "ExtendedS3DestinationConfiguration": {
+ "RoleARN": "arn:aws:iam::000000000000:role/firehose-delivery-role",
+ "BucketARN": "arn:aws:s3:::%s",
+ "BufferingHints": {"SizeInMBs": 5, "IntervalInSeconds": 60}
+ }
+ }
+ """.formatted(deliveryStream, sourceArn, bucket))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DeliveryStreamARN", notNullValue());
+
+ // Give the 1s poller a couple of real ticks to run against the pre-loaded stream
+ // before adding a record it is meant to pick up. A backfill bug would buffer
+ // "old" here already; the buffering interval below (60s of the frozen test clock)
+ // is what lets this test tell "polled and buffered" apart from "never polled".
+ Thread.sleep(2500);
+
+ putKinesisRecord(sourceStream, "new");
+
+ // Let a poll tick actually buffer "new" before the clock moves: bufferSince is
+ // stamped from this same frozen Clock, so advancing first would stamp it already
+ // in the future and the interval trigger below would never fire.
+ Thread.sleep(2500);
+
+ // The frozen test-scope Clock only elapses when advanced; the real 1s tick then
+ // finds the buffer due, same as intervalTriggerDeliversAfterIntervalInSecondsElapses.
+ clock.advance(Duration.ofSeconds(61));
+
+ await().atMost(Duration.ofSeconds(15)).untilAsserted(() ->
+ given()
+ .when()
+ .get("/" + bucket + "/" + firstDeliveredKey(bucket))
+ .then()
+ .statusCode(200)
+ .body(equalTo("new\n")));
+ }
+
+ private void createKinesisStream(String streamName) {
+ given()
+ .header("X-Amz-Target", "Kinesis_20131202.CreateStream")
+ .contentType(CONTENT_TYPE)
+ .body("{\"StreamName\": \"" + streamName + "\", \"ShardCount\": 1}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ }
+
+ private void putKinesisRecord(String streamName, String data) {
+ given()
+ .header("X-Amz-Target", "Kinesis_20131202.PutRecord")
+ .contentType(CONTENT_TYPE)
+ .body("{\"StreamName\": \"" + streamName + "\", \"Data\": \""
+ + Base64.getEncoder().encodeToString(data.getBytes(StandardCharsets.UTF_8))
+ + "\", \"PartitionKey\": \"pk\"}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("SequenceNumber", notNullValue());
+ }
+
private void createDeliveryStream(String streamName, String bucket, int sizeInMBs, int intervalInSeconds) {
given()
.contentType(CONTENT_TYPE)
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
index 0b24b91a96..0fa591423f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
@@ -7,6 +7,7 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
import io.github.hectorvent.floci.services.firehose.model.Record;
+import io.github.hectorvent.floci.services.kinesis.KinesisService;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.PutObjectOptions;
import io.github.hectorvent.floci.testing.MutableClock;
@@ -21,6 +22,8 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -43,6 +46,7 @@ class FirehoseServiceTest {
private static final String FIVE_RECORDS = "{\"n\":0}\n{\"n\":1}\n{\"n\":2}\n{\"n\":3}\n{\"n\":4}\n";
private FirehoseService firehoseService;
+ private KinesisService kinesisService;
private StorageFactory storageFactory;
private S3Service s3Service;
private MutableClock clock;
@@ -50,8 +54,16 @@ class FirehoseServiceTest {
@BeforeEach
void setUp() {
storageFactory = Mockito.mock(StorageFactory.class);
+ // A backend per store, keyed by file name, exactly as the real StorageFactory
+ // reuses backends by path. One shared instance would put KinesisStream values in
+ // the map FirehoseService.scan() reads as delivery streams; a fresh instance per
+ // create() call would make a second service over "the same storage" impossible
+ // to build, which is what the restart test needs.
+ Map<String, AccountAwareStorageBackend<?>> backends = new HashMap<>();
when(storageFactory.create(anyString(), anyString(), any()))
- .thenReturn(AccountAwareStorageBackend.inMemory("000000000000"));
+ .thenAnswer(invocation -> backends.computeIfAbsent(
+ invocation.getArgument(0) + "/" + invocation.getArgument(1),
+ k -> AccountAwareStorageBackend.inMemory("000000000000")));
s3Service = Mockito.mock(S3Service.class);
clock = new MutableClock();
firehoseService = newService(0);
@@ -67,8 +79,12 @@ class FirehoseServiceTest {
EmulatorConfig config = mock(EmulatorConfig.class);
when(config.services()).thenReturn(servicesCfg);
- return new FirehoseService(storageFactory, s3Service,
- new RegionResolver("us-east-1", "000000000000"), clock, config);
+ RegionResolver regionResolver = new RegionResolver("us-east-1", "000000000000");
+ // A real KinesisService, not a mock: the point of the source poller is that it
+ // sees what a GetRecords consumer sees, and a stubbed one could not show that.
+ kinesisService = new KinesisService(storageFactory, regionResolver);
+ return new FirehoseService(storageFactory, s3Service, kinesisService,
+ regionResolver, clock, config);
}
private static S3Destination destination(String bucketArn, String compressionFormat) {
@@ -419,4 +435,168 @@ class FirehoseServiceTest {
delivered("floci-firehose-results").body());
}
+
+ // --- Kinesis stream as source -------------------------------------------------
+ //
+ // A KinesisStreamAsSource delivery stream is never given records by PutRecord: on
+ // real AWS the API rejects that, and everything it delivers comes from Firehose
+ // reading the source stream. Recording the source and not reading it makes such a
+ // stream a silent black hole -- create succeeds, describe looks healthy, nothing is
+ // ever delivered.
+
+ private static DeliveryStreamDescription.KinesisStreamSource kinesisSource(String streamName) {
+ DeliveryStreamDescription.KinesisStreamSource source =
+ new DeliveryStreamDescription.KinesisStreamSource();
+ source.setKinesisStreamArn("arn:aws:kinesis:us-east-1:000000000000:stream/" + streamName);
+ source.setRoleArn("arn:aws:iam::000000000000:role/firehose");
+ return source;
+ }
+
+ private void createSourcedDeliveryStream(String deliveryStream, String sourceStream) {
+ kinesisService.createStream(sourceStream, 1, "us-east-1");
+ firehoseService.createDeliveryStream(deliveryStream, destination("arn:aws:s3:::sink", null),
+ java.util.List.of(), "KinesisStreamAsSource", kinesisSource(sourceStream));
+ }
+
+ @Test
+ void kinesisSourceRecordsAreDelivered() {
+ createSourcedDeliveryStream("sourced-stream", "src-stream");
+ kinesisService.putRecord("src-stream", "hello".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");
+
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+
+ assertEquals("hello\n", delivered("sink").text());
+ }
+
+ @Test
+ void kinesisSourceRecordsAreDeliveredExactlyOnce() {
+ createSourcedDeliveryStream("sourced-stream", "src-stream");
+ kinesisService.putRecord("src-stream", "hello".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");
+
+ // The iterator is the checkpoint, so a second poll before the flush must not
+ // re-read what the first one already buffered.
+ firehoseService.pollKinesisSources();
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+
+ assertEquals("hello\n", delivered("sink").text());
+ }
+
+ @Test
+ void kinesisSourcePollingResumesAfterAFlush() {
+ createSourcedDeliveryStream("sourced-stream", "src-stream");
+ kinesisService.putRecord("src-stream", "first".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+
+ kinesisService.putRecord("src-stream", "second".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+
+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service, Mockito.times(2)).putObject(anyString(), anyString(), body.capture(), anyString(),
+ anyMap(), any(PutObjectOptions.class));
+ assertEquals(java.util.List.of("first\n", "second\n"),
+ body.getAllValues().stream().map(b -> new String(b, StandardCharsets.UTF_8)).toList());
+ }
+
+ @Test
+ void kinesisSourceDoesNotBackfillRecordsFromBeforeDeliveryStart() {
+ kinesisService.createStream("src-stream", 1, "us-east-1");
+ kinesisService.putRecord("src-stream", "old".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");
+ DeliveryStreamDescription.KinesisStreamSource source = kinesisSource("src-stream");
+ // Attaching Firehose to a stream that already holds records does not replay them;
+ // delivery starts at DeliveryStartTimestamp.
+ source.setDeliveryStartTimestamp(Instant.now().plusSeconds(3600));
+ firehoseService.createDeliveryStream("sourced-stream", destination("arn:aws:s3:::sink", null),
+ java.util.List.of(), "KinesisStreamAsSource", source);
+
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+
+ verifyNothingDelivered();
+ }
+
+ @Test
+ void deliveryStreamWithNoKinesisSourceIsUnaffectedByPolling() {
+ firehoseService.createDeliveryStream("plain-stream", destination("arn:aws:s3:::sink", null));
+
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("plain-stream");
+
+ verifyNothingDelivered();
+ }
+
+ @Test
+ void kinesisSourceCheckpointSurvivesARestart() {
+ createSourcedDeliveryStream("sourced-stream", "src-stream");
+ kinesisService.putRecord("src-stream", "hello".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+
+ // Restart: a fresh service over the same storage, the way the emulator comes back
+ // up against its persisted state. The delivery stream and the source stream's
+ // records both survive, so an in-memory-only checkpoint would rebuild the iterator
+ // from DeliveryStartTimestamp and deliver "hello" to S3 a second time.
+ firehoseService = newService(0);
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+
+ assertEquals("hello\n", delivered("sink").text());
+ }
+
+ @Test
+ void kinesisSourceRecordsBufferedButNeverFlushedSurviveARestart() {
+ createSourcedDeliveryStream("sourced-stream", "src-stream");
+ kinesisService.putRecord("src-stream", "hello".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");
+
+ // Poll, then crash. The record is in the in-memory buffer and no flush has run,
+ // so nothing reached S3 and nothing in memory survives.
+ firehoseService.pollKinesisSources();
+ verifyNothingDelivered();
+
+ // Restart. A checkpoint persisted at poll time would already have recorded the
+ // record as consumed, and the restored poller would skip past it forever --
+ // silent, permanent loss. Committing the checkpoint only on a successful flush
+ // means the restored poller reads it again and delivers it.
+ firehoseService = newService(0);
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+
+ assertEquals("hello\n", delivered("sink").text());
+ }
+
+ @Test
+ void aFailedDeliveryLeavesTheSourceCheckpointUncommitted() {
+ createSourcedDeliveryStream("sourced-stream", "src-stream");
+ kinesisService.putRecord("src-stream", "hello".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");
+ when(s3Service.putObject(anyString(), anyString(), any(byte[].class), anyString(),
+ anyMap(), any(PutObjectOptions.class))).thenThrow(new RuntimeException("s3 unavailable"));
+
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+
+ // The write failed, so the record is not durable and the checkpoint must not have
+ // moved: once S3 is back, the next poll has to read it again.
+ Mockito.reset(s3Service);
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+
+ assertEquals("hello\n", delivered("sink").text());
+ }
+
+ @Test
+ void aMissingSourceStreamDoesNotStopOtherStreamsFromPolling() {
+ // No createStream for "gone-stream": describeStream throws ResourceNotFound.
+ firehoseService.createDeliveryStream("broken-stream", destination("arn:aws:s3:::sink", null),
+ java.util.List.of(), "KinesisStreamAsSource", kinesisSource("gone-stream"));
+ createSourcedDeliveryStream("sourced-stream", "src-stream");
+ kinesisService.putRecord("src-stream", "hello".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");
+
+ firehoseService.pollKinesisSources();
+ firehoseService.flush("sourced-stream");
+
+ assertEquals("hello\n", delivered("sink").text());
+ }
}