Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(firehose): honor BufferingHints with time and size based flush (#2314)

Mihai Blidarubfda938f7223
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java
index a249444f6c..ee730a3bd8 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java
@@ -9,6 +9,9 @@ import software.amazon.awssdk.services.firehose.model.PutRecordRequest;
import software.amazon.awssdk.services.firehose.model.Record;
import software.amazon.awssdk.services.glue.GlueClient;
import software.amazon.awssdk.services.glue.model.*;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
@@ -26,6 +29,7 @@ class DataLakeTest {
private static AthenaClient athena;
private static GlueClient glue;
private static FirehoseClient firehose;
+ private static S3Client s3;
private static final String DB_NAME = TestFixtures.uniqueName("test_db");
private static final String TABLE_NAME = "orders";
@@ -36,6 +40,7 @@ class DataLakeTest {
athena = TestFixtures.athenaClient();
glue = TestFixtures.glueClient();
firehose = TestFixtures.firehoseClient();
+ s3 = TestFixtures.s3Client();
}
@Test
@@ -76,6 +81,12 @@ class DataLakeTest {
.bucketARN("arn:aws:s3:::floci-firehose-results")
.roleARN("arn:aws:iam::000000000000:role/datalake-firehose-role")
.prefix(STREAM_NAME + "/")
+ // Floci does not enforce AWS's 60s minimum interval, which keeps
+ // the delivery wait short (would need 60+ against real AWS).
+ .bufferingHints(software.amazon.awssdk.services.firehose.model.BufferingHints.builder()
+ .sizeInMBs(1)
+ .intervalInSeconds(5)
+ .build())
.build())
.build());
}
@@ -92,6 +103,28 @@ class DataLakeTest {
.build());
}
+ // Small records stay buffered until the stream's IntervalInSeconds
+ // elapses; wait for Firehose to deliver before querying, the same way a
+ // real AWS client would.
+ long deadline = System.currentTimeMillis() + 30_000;
+ boolean delivered = false;
+ while (!delivered && System.currentTimeMillis() < deadline) {
+ try {
+ delivered = !s3.listObjectsV2(ListObjectsV2Request.builder()
+ .bucket("floci-firehose-results")
+ .prefix(STREAM_NAME + "/")
+ .build()).contents().isEmpty();
+ } catch (NoSuchBucketException e) {
+ // The bucket itself is only created on the first delivery.
+ }
+ if (!delivered) {
+ Thread.sleep(2_000);
+ }
+ }
+ assertThat(delivered)
+ .as("Firehose should deliver the buffered records within IntervalInSeconds")
+ .isTrue();
+
// Athena Query
StartQueryExecutionResponse startResp = athena.startQueryExecution(StartQueryExecutionRequest.builder()
.queryString("SELECT sum(amount) as total FROM " + TABLE_NAME)
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.java
index 6334012bde..07130e0ca6 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.java
@@ -56,6 +56,7 @@ import software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest;
import software.amazon.awssdk.services.sqs.model.ReceiveMessageResponse;
import software.amazon.awssdk.services.firehose.FirehoseClient;
+import software.amazon.awssdk.services.firehose.model.BufferingHints;
import software.amazon.awssdk.services.firehose.model.CreateDeliveryStreamRequest;
import software.amazon.awssdk.services.firehose.model.DeleteDeliveryStreamRequest;
import software.amazon.awssdk.services.firehose.model.S3DestinationConfiguration;
@@ -199,6 +200,12 @@ class SesEventPublishingTest {
.bucketARN("arn:aws:s3:::" + firehoseBucket)
.roleARN("arn:aws:iam::000000000000:role/sdk-evt-fh-role")
.prefix(firehoseStreamName + "/")
+ // Floci does not enforce AWS's 60s minimum interval, which keeps
+ // the delivery wait short (would need 60+ against real AWS).
+ .bufferingHints(BufferingHints.builder()
+ .sizeInMBs(1)
+ .intervalInSeconds(5)
+ .build())
.build())
.build())
.deliveryStreamARN();
@@ -478,9 +485,9 @@ class SesEventPublishingTest {
@Test
@Order(5)
- @DisplayName("Firehose destination: 5 sends trigger auto-flush to S3 as NDJSON")
- void firehoseDestination_fiveSends_triggerAutoFlushToS3() throws Exception {
- for (int i = 0; i < 5; i++) {
+ @DisplayName("Firehose destination: send events are delivered to S3 as NDJSON within the buffering interval")
+ void firehoseDestination_sendEvents_deliveredWithinBufferingInterval() throws Exception {
+ for (int i = 0; i < 2; i++) {
ses.sendEmail(SendEmailRequest.builder()
.fromEmailAddress(identity)
.destination(Destination.builder()
@@ -498,12 +505,22 @@ class SesEventPublishingTest {
.build());
}
- ListObjectsV2Response listed = s3.listObjectsV2(ListObjectsV2Request.builder()
- .bucket(firehoseBucket)
- .prefix(firehoseStreamName + "/")
- .build());
+ // Two small events stay well below SizeInMBs, so delivery happens on the
+ // stream's IntervalInSeconds plus the emulator's flush tick.
+ ListObjectsV2Response listed = null;
+ long deadline = System.currentTimeMillis() + 30_000;
+ while (System.currentTimeMillis() < deadline) {
+ listed = s3.listObjectsV2(ListObjectsV2Request.builder()
+ .bucket(firehoseBucket)
+ .prefix(firehoseStreamName + "/")
+ .build());
+ if (!listed.contents().isEmpty()) {
+ break;
+ }
+ Thread.sleep(2_000);
+ }
assertThat(listed.contents())
- .as("Firehose should have flushed exactly one S3 object after 5 records")
+ .as("Firehose should deliver the buffered events within IntervalInSeconds")
.hasSize(1);
S3Object obj = listed.contents().get(0);
@@ -511,7 +528,7 @@ class SesEventPublishingTest {
.bucket(firehoseBucket).key(obj.key()).build())
.readAllBytes(), StandardCharsets.UTF_8);
String[] lines = body.split("\\R");
- assertThat(lines).hasSize(5);
+ assertThat(lines).hasSize(2);
for (String line : lines) {
JsonNode event = MAPPER.readTree(line);
assertThat(event.path("eventType").asText()).isEqualTo("Send");
diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md
index 7762a50b63..f4ef6860fe 100644
--- a/docs/configuration/environment-variables.md
+++ b/docs/configuration/environment-variables.md
@@ -232,6 +232,8 @@ See [Initialization Hooks](./initialization-hooks.md) for lifecycle phases and s
| Variable | Default | Description |
|---|---|---|
| `FLOCI_SERVICES_FIREHOSE_ENABLED` | `true` | Enable the Kinesis Data Firehose service |
+| `FLOCI_SERVICES_FIREHOSE_TICK_INTERVAL_SECONDS` | `10` | How often (seconds) the buffer flusher checks for streams whose `BufferingHints.IntervalInSeconds` has elapsed |
+| `FLOCI_SERVICES_FIREHOSE_FLUSH_RECORD_COUNT` | `0` | Emulator-only: flush after this many buffered records (`0` = disabled, AWS-faithful; `1` = LocalStack-style record-at-a-time delivery) |
### EventBridge
diff --git a/docs/services/firehose.md b/docs/services/firehose.md
index 5d96befbb5..d3ceeb1661 100644
--- a/docs/services/firehose.md
+++ b/docs/services/firehose.md
@@ -25,7 +25,13 @@ Floci emulates Amazon Data Firehose for streaming data ingestion and delivery to
## How it works
1. **Buffering**: Incoming records are buffered in memory.
-2. **Automatic Flush**: Floci automatically flushes the buffer to S3 after every 5 records for immediate local feedback.
+2. **Automatic Flush**: A buffer is delivered to S3 when either trigger of the stream's `BufferingHints` fires first, mirroring AWS's buffered delivery:
+ - the buffered records reach `SizeInMBs` (default 5 MiB), or
+ - `IntervalInSeconds` (default 300 s) has elapsed since the first buffered record. A background flusher checks this every `floci.services.firehose.tick-interval-seconds` (default 10 s).
+
+ An emulator-only record-count trigger is also available for local dev: set `floci.services.firehose.flush-record-count` (env: `FLOCI_SERVICES_FIREHOSE_FLUSH_RECORD_COUNT`) to flush after that many buffered records — `1` gives LocalStack-style record-at-a-time delivery. Disabled by default (`0`) so delivery timing matches real AWS.
+
+ Buffers are also flushed on shutdown, so pending records are not lost. Deleting a delivery stream discards its pending records without flushing, matching real AWS.
3. **Format**: Records are flushed as raw NDJSON (newline-delimited JSON) to the bucket configured in the S3 destination (`floci-firehose-results` if the stream has no destination configuration).
## S3 object keys
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 6d45d36839..8b4580ea04 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -1083,6 +1083,22 @@ public interface EmulatorConfig {
interface FirehoseServiceConfig {
@WithDefault("true")
boolean enabled();
+
+ /**
+ * How often the buffer flusher checks for streams whose buffering
+ * interval (BufferingHints.IntervalInSeconds) has elapsed.
+ */
+ @WithDefault("10")
+ long tickIntervalSeconds();
+
+ /**
+ * Emulator-only volume trigger: number of buffered records that forces
+ * an immediate flush, complementing the stream's BufferingHints.
+ * Disabled by default (0) so out-of-the-box delivery matches real AWS;
+ * set to 1 for LocalStack-style record-at-a-time delivery in local dev.
+ */
+ @WithDefault("0")
+ int flushRecordCount();
}
interface KmsServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
index 0577c0353d..57b3cd2149 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.firehose;
import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
@@ -11,37 +12,114 @@ import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescript
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
import io.github.hectorvent.floci.services.firehose.model.Record;
import io.github.hectorvent.floci.services.s3.S3Service;
+import io.quarkus.runtime.ShutdownDelayInitiatedEvent;
+import io.quarkus.runtime.StartupEvent;
import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.event.Observes;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.nio.charset.StandardCharsets;
import java.time.Clock;
+import java.time.Instant;
import java.time.ZoneId;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
@ApplicationScoped
public class FirehoseService {
private static final Logger LOG = Logger.getLogger(FirehoseService.class);
private static final String DEFAULT_BUCKET = "floci-firehose-results";
- private static final int DEFAULT_FLUSH_COUNT = 5;
+ private static final int DEFAULT_BUFFERING_INTERVAL_SECONDS = 300;
+ private static final int DEFAULT_BUFFERING_SIZE_MBS = 5;
private final StorageBackend<String, DeliveryStreamDescription> streamStore;
private final Map<String, List<byte[]>> buffers = new ConcurrentHashMap<>();
+ private final Map<String, Instant> bufferSince = new ConcurrentHashMap<>();
private final S3Service s3Service;
private final RegionResolver regionResolver;
private final Clock clock;
+ private final long tickIntervalSeconds;
+ private final int flushRecordCount;
+ private final boolean flusherEnabled;
+ private final ScheduledExecutorService flushExecutor;
@Inject
public FirehoseService(StorageFactory storageFactory, S3Service s3Service, RegionResolver regionResolver,
- Clock clock) {
+ Clock clock, EmulatorConfig config) {
this.streamStore = storageFactory.create("firehose", "streams.json",
new TypeReference<Map<String, DeliveryStreamDescription>>() {});
this.s3Service = s3Service;
this.regionResolver = regionResolver;
this.clock = clock;
+ this.tickIntervalSeconds = Math.max(1, config.services().firehose().tickIntervalSeconds());
+ this.flushRecordCount = Math.max(0, config.services().firehose().flushRecordCount());
+ this.flusherEnabled = config.services().firehose().enabled();
+ this.flushExecutor = Executors.newSingleThreadScheduledExecutor(r -> {
+ Thread t = new Thread(r, "firehose-buffer-flusher");
+ t.setDaemon(true);
+ return t;
+ });
+ }
+
+ void onStart(@Observes StartupEvent ignored) {
+ if (!flusherEnabled) {
+ LOG.info("Firehose buffer flusher disabled by configuration");
+ return;
+ }
+ flushExecutor.scheduleAtFixedRate(this::tickSafely, tickIntervalSeconds, tickIntervalSeconds, TimeUnit.SECONDS);
+ LOG.infov("Firehose buffer flusher started (tick every {0}s)", tickIntervalSeconds);
+ }
+
+ // ShutdownDelayInitiatedEvent fires before every ShutdownEvent observer, so this
+ // drain lands the pending records in S3 while EmulatorLifecycle.onStop can still
+ // persist them to disk via storageFactory.flushAll().
+ void onPreShutdown(@Observes ShutdownDelayInitiatedEvent ignored) {
+ flushExecutor.shutdownNow();
+ buffers.keySet().forEach(this::flush);
+ }
+
+ void tickSafely() {
+ try {
+ flushDueBuffers(clock.instant());
+ } catch (Throwable t) {
+ LOG.warnv("Firehose buffer flush tick failed: {0}", t.getMessage());
+ }
+ }
+
+ void flushDueBuffers(Instant now) {
+ for (Map.Entry<String, List<byte[]>> entry : buffers.entrySet()) {
+ if (entry.getValue().isEmpty()) {
+ continue;
+ }
+ String streamName = entry.getKey();
+ try {
+ DeliveryStreamDescription stream = describeDeliveryStream(streamName);
+ Instant since = bufferSince.putIfAbsent(streamName, now);
+ if (since == null) {
+ since = now;
+ }
+ if (!now.isBefore(since.plusSeconds(bufferingIntervalSeconds(stream)))) {
+ flush(streamName, stream);
+ }
+ } catch (Exception e) {
+ LOG.warnv("Firehose buffer flush failed for stream {0}: {1}", streamName, e.getMessage());
+ }
+ }
+ }
+
+ private static int bufferingIntervalSeconds(DeliveryStreamDescription stream) {
+ S3Destination s3 = stream.s3Destination();
+ // describeDeliveryStream already applied defaults, so hints only miss
+ // when the stream has no S3 destination at all (default-bucket delivery).
+ if (s3 == null || s3.getBufferingHints() == null || s3.getBufferingHints().getIntervalInSeconds() == null) {
+ return DEFAULT_BUFFERING_INTERVAL_SECONDS;
+ }
+ return s3.getBufferingHints().getIntervalInSeconds();
}
public String createDeliveryStream(String name, S3Destination s3Config) {
@@ -219,7 +297,11 @@ public class FirehoseService {
public void deleteDeliveryStream(String name) {
describeDeliveryStream(name);
streamStore.delete(name);
+ // Pending records are discarded, not flushed: verified against real AWS
+ // (2026-07-13, eu-west-1) — 3 records buffered under a 300s/5MB hint never
+ // reached the bucket after DeleteDeliveryStream completed.
buffers.remove(name);
+ bufferSince.remove(name);
LOG.infov("Deleted Firehose delivery stream: {0}", name);
}
@@ -229,27 +311,41 @@ public class FirehoseService {
}
public void putRecord(String streamName, Record record) {
- DeliveryStreamDescription stream = describeDeliveryStream(streamName);
- buffers.computeIfAbsent(streamName, k -> Collections.synchronizedList(new ArrayList<>()))
- .add(record.getData());
-
- if (buffers.get(streamName).size() >= DEFAULT_FLUSH_COUNT) {
- flush(streamName, stream);
- }
+ putRecordBatch(streamName, List.of(record));
}
public void putRecordBatch(String streamName, List<Record> records) {
DeliveryStreamDescription stream = describeDeliveryStream(streamName);
List<byte[]> buffer = buffers.computeIfAbsent(
streamName, k -> Collections.synchronizedList(new ArrayList<>()));
- for (Record r : records) {
- buffer.add(r.getData());
+ long bufferedBytes = 0;
+ int bufferedCount;
+ // Records and their buffering-start timestamp move together under the
+ // buffer lock so the flusher never sees one without the other.
+ synchronized (buffer) {
+ for (Record r : records) {
+ buffer.add(r.getData());
+ }
+ bufferSince.putIfAbsent(streamName, clock.instant());
+ bufferedCount = buffer.size();
+ for (byte[] data : buffer) {
+ bufferedBytes += data.length;
+ }
}
- if (buffer.size() >= DEFAULT_FLUSH_COUNT) {
+ if ((flushRecordCount > 0 && bufferedCount >= flushRecordCount)
+ || bufferedBytes >= bufferingSizeLimitBytes(stream)) {
flush(streamName, stream);
}
}
+ private static long bufferingSizeLimitBytes(DeliveryStreamDescription stream) {
+ S3Destination s3 = stream.s3Destination();
+ int sizeInMBs = (s3 == null || s3.getBufferingHints() == null || s3.getBufferingHints().getSizeInMBs() == null)
+ ? DEFAULT_BUFFERING_SIZE_MBS
+ : s3.getBufferingHints().getSizeInMBs();
+ return sizeInMBs * 1024L * 1024L;
+ }
+
public void flush(String streamName) {
streamStore.get(streamName).ifPresent(stream -> flush(streamName, stream));
}
@@ -264,6 +360,11 @@ public class FirehoseService {
synchronized (buffer) {
toFlush = new ArrayList<>(buffer);
buffer.clear();
+ bufferSince.remove(streamName);
+ }
+ if (toFlush.isEmpty()) {
+ // Lost the race against a concurrent flush; nothing left to deliver.
+ return;
}
try {
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 95de580ceb..280c84b52b 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -312,6 +312,8 @@ floci:
enabled: true
firehose:
enabled: true
+ tick-interval-seconds: 10
+ flush-record-count: 0
kms:
enabled: true
cognito:
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java
index 406471215d..e29d267bab 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.firehose;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
@@ -20,6 +21,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
class FirehoseExtendedS3IntegrationTest {
+ @Inject
+ FirehoseService firehoseService;
+
private static final String STREAM_NAME = "test-extended-s3-stream";
private static final String LEGACY_STREAM_NAME = "test-legacy-s3-stream";
private static final String MINIMAL_STREAM_NAME = "test-minimal-extended-s3-stream";
@@ -339,7 +343,7 @@ class FirehoseExtendedS3IntegrationTest {
String stream = "delivery-default-stream";
String bucket = "extended-s3-delivery-default";
createDeliveryStream(stream, bucket, "");
- putFiveRecords(stream);
+ putFiveRecordsAndFlush(stream);
String key = firstDeliveredKey(bucket, "");
assertTrue(key.matches(TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);
@@ -351,7 +355,7 @@ class FirehoseExtendedS3IntegrationTest {
String stream = "delivery-static-stream";
String bucket = "extended-s3-delivery-static";
createDeliveryStream(stream, bucket, ", \"Prefix\": \"events/data/\"");
- putFiveRecords(stream);
+ putFiveRecordsAndFlush(stream);
String key = firstDeliveredKey(bucket, "events/data/");
assertTrue(key.matches("events/data/" + TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);
@@ -363,7 +367,7 @@ class FirehoseExtendedS3IntegrationTest {
String stream = "delivery-noslash-stream";
String bucket = "extended-s3-delivery-noslash";
createDeliveryStream(stream, bucket, ", \"Prefix\": \"legacy\"");
- putFiveRecords(stream);
+ putFiveRecordsAndFlush(stream);
String key = firstDeliveredKey(bucket, "legacy");
assertTrue(key.matches("legacy" + TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);
@@ -375,7 +379,7 @@ class FirehoseExtendedS3IntegrationTest {
String stream = "delivery-expr-stream";
String bucket = "extended-s3-delivery-expr";
createDeliveryStream(stream, bucket, ", \"Prefix\": \"data/!{timestamp:yyyy/MM/dd}/\"");
- putFiveRecords(stream);
+ putFiveRecordsAndFlush(stream);
String key = firstDeliveredKey(bucket, "data/");
assertTrue(key.matches("data/\\d{4}/\\d{2}/\\d{2}/" + stream + SUFFIX_REGEX), key);
@@ -387,7 +391,7 @@ class FirehoseExtendedS3IntegrationTest {
String stream = "delivery-rand-stream";
String bucket = "extended-s3-delivery-rand";
createDeliveryStream(stream, bucket, ", \"Prefix\": \"rand/!{firehose:random-string}/\"");
- putFiveRecords(stream);
+ putFiveRecordsAndFlush(stream);
String key = firstDeliveredKey(bucket, "rand/");
assertTrue(key.matches("rand/[A-Za-z0-9]{11}/" + stream + SUFFIX_REGEX), key);
@@ -412,7 +416,7 @@ class FirehoseExtendedS3IntegrationTest {
.body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CustomTimeZone",
equalTo("Europe/Madrid"));
- putFiveRecords(stream);
+ putFiveRecordsAndFlush(stream);
String key = firstDeliveredKey(bucket, "events/");
assertTrue(key.matches("events/" + TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);
}
@@ -436,8 +440,12 @@ class FirehoseExtendedS3IntegrationTest {
.statusCode(200);
}
- /** Five records reach DEFAULT_FLUSH_COUNT, so the batch triggers the automatic flush. */
- private void putFiveRecords(String streamName) {
+ /**
+ * Small records stay buffered until the size (SizeInMBs) or interval
+ * (IntervalInSeconds) trigger fires; force the flush so the delivery
+ * assertions are deterministic (same mechanism the scheduled flusher uses).
+ */
+ private void putFiveRecordsAndFlush(String streamName) {
String data = Base64.getEncoder().encodeToString("{\"id\": 1}".getBytes(StandardCharsets.UTF_8));
given()
.contentType(CONTENT_TYPE)
@@ -455,6 +463,7 @@ class FirehoseExtendedS3IntegrationTest {
.then()
.statusCode(200)
.body("FailedPutCount", equalTo(0));
+ firehoseService.flush(streamName);
}
private String firstDeliveredKey(String bucket, String listPrefix) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushIntegrationTest.java
new file mode 100644
index 0000000000..267c57fdc6
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushIntegrationTest.java
@@ -0,0 +1,149 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import io.github.hectorvent.floci.testing.MutableClock;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.Base64;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static io.restassured.RestAssured.given;
+import static org.awaitility.Awaitility.await;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Wire-level coverage of the AWS-faithful BufferingHints flush triggers:
+ * volume ({@code SizeInMBs}) and time ({@code IntervalInSeconds}), delivered
+ * by the background flusher without any explicit flush call.
+ *
+ * The emulator-only record-count trigger is covered by
+ * {@link FirehoseFlushRecordCountIntegrationTest}: it needs a different
+ * config override, and a Quarkus test profile applies to the whole class.
+ */
+@QuarkusTest
+@TestProfile(FirehoseFlushIntegrationTest.FastTickProfile.class)
+class FirehoseFlushIntegrationTest {
+
+ public static class FastTickProfile implements QuarkusTestProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ // 1s flush tick so the interval test waits seconds, not the 10s default.
+ return Map.of("floci.services.firehose.tick-interval-seconds", "1");
+ }
+ }
+
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final String TARGET_PREFIX = "Firehose_20150804.";
+ private static final Pattern KEY_PATTERN = Pattern.compile("<Key>([^<]+)</Key>");
+
+ /** The global test-scope Clock alternative, frozen at 2026-01-01 unless advanced. */
+ @Inject
+ MutableClock clock;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void sizeTriggerDeliversWhenBufferedBytesReachSizeInMBs() {
+ String stream = "flush-size-stream";
+ String bucket = "flush-size-archive";
+ createDeliveryStream(stream, bucket, 1, 300);
+
+ String halfMiB = Base64.getEncoder()
+ .encodeToString("A".repeat(512 * 1024).getBytes(StandardCharsets.UTF_8));
+ putRecord(stream, halfMiB);
+ putRecord(stream, halfMiB); // crosses the 1 MiB threshold: flushes inline, no waiting
+
+ String body = given()
+ .when()
+ .get("/" + bucket + "/" + firstDeliveredKey(bucket))
+ .then()
+ .statusCode(200)
+ .extract().asString();
+ // Both records, each followed by the newline the flush appends.
+ assertEquals(2 * 512 * 1024 + 2, body.length());
+ assertTrue(body.startsWith("A"), "delivered body should contain the record payload");
+ }
+
+ @Test
+ void intervalTriggerDeliversAfterIntervalInSecondsElapses() {
+ String stream = "flush-interval-stream";
+ String bucket = "flush-interval-archive";
+ createDeliveryStream(stream, bucket, 5, 60);
+
+ putRecord(stream, Base64.getEncoder()
+ .encodeToString("{\"n\":42}".getBytes(StandardCharsets.UTF_8)));
+
+ // The test-scope Clock is frozen, so the interval can only elapse by
+ // advancing it; the real 1s tick then finds the buffer due.
+ clock.advance(Duration.ofSeconds(61));
+
+ await().atMost(Duration.ofSeconds(15)).untilAsserted(() ->
+ given()
+ .when()
+ .get("/" + bucket + "/" + firstDeliveredKey(bucket))
+ .then()
+ .statusCode(200)
+ .body(equalTo("{\"n\":42}\n")));
+ }
+
+ private void createDeliveryStream(String streamName, String bucket, int sizeInMBs, int intervalInSeconds) {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "DeliveryStreamType": "DirectPut",
+ "ExtendedS3DestinationConfiguration": {
+ "RoleARN": "arn:aws:iam::000000000000:role/firehose-delivery-role",
+ "BucketARN": "arn:aws:s3:::%s",
+ "BufferingHints": {"SizeInMBs": %d, "IntervalInSeconds": %d}
+ }
+ }
+ """.formatted(streamName, bucket, sizeInMBs, intervalInSeconds))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DeliveryStreamARN", notNullValue());
+ }
+
+ private void putRecord(String streamName, String base64Data) {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "PutRecord")
+ .body("{ \"DeliveryStreamName\": \"" + streamName + "\", \"Record\": {\"Data\": \"" + base64Data + "\"} }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("RecordId", notNullValue());
+ }
+
+ private String firstDeliveredKey(String bucket) {
+ String listing = given()
+ .when()
+ .get("/" + bucket)
+ .then()
+ .statusCode(200)
+ .extract().asString();
+ Matcher key = KEY_PATTERN.matcher(listing);
+ assertTrue(key.find(), "expected a delivered object in " + bucket + ", got: " + listing);
+ return key.group(1);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushRecordCountIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushRecordCountIntegrationTest.java
new file mode 100644
index 0000000000..cb080c03a7
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushRecordCountIntegrationTest.java
@@ -0,0 +1,109 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Verifies {@code floci.services.firehose.flush-record-count=1} delivers
+ * every record to the S3 destination immediately.
+ *
+ * Kept separate from {@link FirehoseFlushIntegrationTest} (size and interval
+ * triggers) because this class needs a different config override and a
+ * Quarkus test profile applies to the whole class.
+ */
+@QuarkusTest
+@TestProfile(FirehoseFlushRecordCountIntegrationTest.FlushEveryRecordProfile.class)
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class FirehoseFlushRecordCountIntegrationTest {
+
+ public static class FlushEveryRecordProfile implements QuarkusTestProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return Map.of("floci.services.firehose.flush-record-count", "1");
+ }
+ }
+
+ private static final String STREAM_NAME = "test-flush-count-stream";
+ private static final String BUCKET = "flush-count-archive";
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final String TARGET_PREFIX = "Firehose_20150804.";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(1)
+ void createDeliveryStream() {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "DeliveryStreamType": "DirectPut",
+ "ExtendedS3DestinationConfiguration": {
+ "RoleARN": "arn:aws:iam::000000000000:role/firehose-delivery-role",
+ "BucketARN": "arn:aws:s3:::%s",
+ "Prefix": "single/"
+ }
+ }
+ """.formatted(STREAM_NAME, BUCKET))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DeliveryStreamARN", notNullValue());
+ }
+
+ @Test
+ @Order(2)
+ void singleRecordIsDeliveredImmediately() {
+ // {"n":1} — base64: eyJuIjoxfQ==
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "PutRecord")
+ .body("{ \"DeliveryStreamName\": \"" + STREAM_NAME + "\", \"Record\": {\"Data\": \"eyJuIjoxfQ==\"} }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("RecordId", notNullValue());
+
+ // The record must be visible in the destination bucket without any
+ // further puts (path-style S3 ListObjects on the same edge port).
+ String listing = given()
+ .when()
+ .get("/" + BUCKET)
+ .then()
+ .statusCode(200)
+ .extract().asString();
+
+ Matcher key = Pattern.compile("<Key>(single/[^<]+)</Key>").matcher(listing);
+ assertTrue(key.find(), "expected a delivered object under single/, got: " + listing);
+
+ given()
+ .when()
+ .get("/" + BUCKET + "/" + key.group(1))
+ .then()
+ .statusCode(200)
+ .body(equalTo("{\"n\":1}\n"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
index b83fbeefe2..13357d3e89 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
@@ -1,7 +1,7 @@
package io.github.hectorvent.floci.services.firehose;
+import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.RegionResolver;
-import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;
@@ -15,12 +15,16 @@ import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import java.nio.charset.StandardCharsets;
+import java.time.Instant;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyMap;
import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -29,25 +33,46 @@ class FirehoseServiceTest {
private static final String UUID_REGEX = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
private FirehoseService firehoseService;
+ private StorageFactory storageFactory;
private S3Service s3Service;
+ private MutableClock clock;
@BeforeEach
void setUp() {
- StorageFactory storageFactory = Mockito.mock(StorageFactory.class);
+ storageFactory = Mockito.mock(StorageFactory.class);
when(storageFactory.create(anyString(), anyString(), any()))
.thenReturn(AccountAwareStorageBackend.inMemory("000000000000"));
s3Service = Mockito.mock(S3Service.class);
- firehoseService = new FirehoseService(storageFactory, s3Service,
- new RegionResolver("us-east-1", "000000000000"), new MutableClock());
+ clock = new MutableClock();
+ firehoseService = newService(0);
}
- private void putRecordsUntilFlush(String streamName) {
- for (int i = 0; i < 5; i++) {
- Record record = new Record(("{\"n\":" + i + "}").getBytes(StandardCharsets.UTF_8));
- firehoseService.putRecord(streamName, record);
+ private FirehoseService newService(int flushRecordCount) {
+ EmulatorConfig.FirehoseServiceConfig firehoseCfg = mock(EmulatorConfig.FirehoseServiceConfig.class);
+ when(firehoseCfg.enabled()).thenReturn(true);
+ when(firehoseCfg.tickIntervalSeconds()).thenReturn(10L);
+ when(firehoseCfg.flushRecordCount()).thenReturn(flushRecordCount);
+ EmulatorConfig.ServicesConfig servicesCfg = mock(EmulatorConfig.ServicesConfig.class);
+ when(servicesCfg.firehose()).thenReturn(firehoseCfg);
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ when(config.services()).thenReturn(servicesCfg);
+
+ return new FirehoseService(storageFactory, s3Service,
+ new RegionResolver("us-east-1", "000000000000"), clock, config);
+ }
+
+ private void putRecords(String streamName, int count) {
+ for (int i = 0; i < count; i++) {
+ firehoseService.putRecord(streamName, new Record(("{\"n\":" + i + "}").getBytes(StandardCharsets.UTF_8)));
}
}
+ /** Puts a few small records and forces delivery, the way the interval trigger eventually would. */
+ private void putRecordsAndFlush(String streamName) {
+ putRecords(streamName, 5);
+ firehoseService.flush(streamName);
+ }
+
private String deliveredKey(String expectedBucket) {
ArgumentCaptor<String> bucket = ArgumentCaptor.forClass(String.class);
ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);
@@ -59,7 +84,7 @@ class FirehoseServiceTest {
@Test
void deliversToDefaultBucketWithAwsShapedKey() {
firehoseService.createDeliveryStream("my-stream", null);
- putRecordsUntilFlush("my-stream");
+ putRecordsAndFlush("my-stream");
String key = deliveredKey("floci-firehose-results");
assertTrue(key.matches("2026/01/01/00/my-stream-1-2026-01-01-00-00-00-" + UUID_REGEX), key);
@@ -71,7 +96,7 @@ class FirehoseServiceTest {
s3.setBucketArn("arn:aws:s3:::custom-bucket");
s3.setPrefix("events/data/");
firehoseService.createDeliveryStream("my-stream", s3);
- putRecordsUntilFlush("my-stream");
+ putRecordsAndFlush("my-stream");
String key = deliveredKey("custom-bucket");
assertTrue(key.matches("events/data/2026/01/01/00/my-stream-1-2026-01-01-00-00-00-" + UUID_REGEX), key);
@@ -83,7 +108,7 @@ class FirehoseServiceTest {
s3.setBucketArn("arn:aws:s3:::custom-bucket");
s3.setCustomTimeZone("Europe/Madrid");
firehoseService.createDeliveryStream("my-stream", s3);
- putRecordsUntilFlush("my-stream");
+ putRecordsAndFlush("my-stream");
String key = deliveredKey("custom-bucket");
assertTrue(key.matches("2026/01/01/01/my-stream-1-2026-01-01-01-00-00-" + UUID_REGEX), key);
@@ -110,8 +135,129 @@ class FirehoseServiceTest {
assertEquals("Asia/Tokyo",
firehoseService.describeDeliveryStream("my-stream").s3Destination().getCustomTimeZone());
- putRecordsUntilFlush("my-stream");
+ putRecordsAndFlush("my-stream");
String key = deliveredKey("custom-bucket");
assertTrue(key.matches("events/2026/01/01/09/my-stream-3-2026-01-01-09-00-00-" + UUID_REGEX), key);
}
+
+ @Test
+ void timeBasedFlushKeepsBufferWhileDefaultIntervalHasNotElapsed() {
+ firehoseService.createDeliveryStream("idle-stream", null);
+ putRecords("idle-stream", 2);
+
+ firehoseService.flushDueBuffers(clock.instant().plusSeconds(299));
+
+ verify(s3Service, never()).putObject(anyString(), anyString(), any(byte[].class), anyString(), anyMap());
+ }
+
+ @Test
+ void timeBasedFlushDeliversBufferedRecordsAfterDefaultInterval() {
+ firehoseService.createDeliveryStream("idle-stream", null);
+ putRecords("idle-stream", 2);
+
+ firehoseService.flushDueBuffers(clock.instant().plusSeconds(300));
+
+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service).putObject(eq("floci-firehose-results"), anyString(), body.capture(), anyString(), anyMap());
+ assertEquals("{\"n\":0}\n{\"n\":1}\n", new String(body.getValue(), StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void timeBasedFlushHonorsStreamBufferingIntervalHint() {
+ S3Destination s3 = new S3Destination();
+ s3.setBucketArn("arn:aws:s3:::custom-bucket");
+ DeliveryStreamDescription.BufferingHints hints = new DeliveryStreamDescription.BufferingHints();
+ hints.setSizeInMBs(5);
+ hints.setIntervalInSeconds(60);
+ s3.setBufferingHints(hints);
+ firehoseService.createDeliveryStream("hinted-stream", s3);
+ putRecords("hinted-stream", 1);
+
+ firehoseService.flushDueBuffers(clock.instant().plusSeconds(59));
+ verify(s3Service, never()).putObject(anyString(), anyString(), any(byte[].class), anyString(), anyMap());
+
+ firehoseService.flushDueBuffers(clock.instant().plusSeconds(60));
+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service).putObject(eq("custom-bucket"), anyString(), body.capture(), anyString(), anyMap());
+ assertEquals("{\"n\":0}\n", new String(body.getValue(), StandardCharsets.UTF_8));
+ }
+
+ /** Matches real AWS: the volume trigger is bytes vs SizeInMBs, never a record count. */
+ @Test
+ void smallRecordsDoNotTriggerSizeBasedFlushRegardlessOfCount() {
+ firehoseService.createDeliveryStream("trickle-stream", null);
+ putRecords("trickle-stream", 20);
+
+ verify(s3Service, never()).putObject(anyString(), anyString(), any(byte[].class), anyString(), anyMap());
+ }
+
+ @Test
+ void sizeBasedFlushDeliversWhenBufferedBytesReachSizeHint() {
+ S3Destination s3 = new S3Destination();
+ s3.setBucketArn("arn:aws:s3:::custom-bucket");
+ DeliveryStreamDescription.BufferingHints hints = new DeliveryStreamDescription.BufferingHints();
+ hints.setSizeInMBs(1);
+ hints.setIntervalInSeconds(300);
+ s3.setBufferingHints(hints);
+ firehoseService.createDeliveryStream("bulky-stream", s3);
+
+ firehoseService.putRecord("bulky-stream", new Record(new byte[512 * 1024]));
+ verify(s3Service, never()).putObject(anyString(), anyString(), any(byte[].class), anyString(), anyMap());
+
+ firehoseService.putRecord("bulky-stream", new Record(new byte[512 * 1024]));
+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service).putObject(eq("custom-bucket"), anyString(), body.capture(), anyString(), anyMap());
+ // Both 512 KiB records, each followed by the newline the flush appends.
+ assertEquals(2 * 512 * 1024 + 2, body.getValue().length);
+ }
+
+ /** Emulator-only opt-in: flush-record-count=1 restores LocalStack-style record-at-a-time delivery. */
+ @Test
+ void flushRecordCountOfOneDeliversEachRecordImmediately() {
+ firehoseService = newService(1);
+ firehoseService.createDeliveryStream("eager-stream", null);
+ firehoseService.putRecord("eager-stream", new Record("{\"n\":0}".getBytes(StandardCharsets.UTF_8)));
+
+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service).putObject(eq("floci-firehose-results"), anyString(), body.capture(), anyString(), anyMap());
+ assertEquals("{\"n\":0}\n", new String(body.getValue(), StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void flushRecordCountThresholdDeliversTheWholeBuffer() {
+ firehoseService = newService(3);
+ firehoseService.createDeliveryStream("counted-stream", null);
+ putRecords("counted-stream", 2);
+ verify(s3Service, never()).putObject(anyString(), anyString(), any(byte[].class), anyString(), anyMap());
+
+ firehoseService.putRecord("counted-stream", new Record("{\"n\":2}".getBytes(StandardCharsets.UTF_8)));
+
+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service).putObject(eq("floci-firehose-results"), anyString(), body.capture(), anyString(), anyMap());
+ assertEquals("{\"n\":0}\n{\"n\":1}\n{\"n\":2}\n", new String(body.getValue(), StandardCharsets.UTF_8));
+ }
+
+ /** Matches real AWS: DeleteDeliveryStream discards undelivered records instead of flushing them. */
+ @Test
+ void deleteDeliveryStreamDiscardsBufferedRecords() {
+ firehoseService.createDeliveryStream("doomed-stream", null);
+ putRecords("doomed-stream", 2);
+
+ firehoseService.deleteDeliveryStream("doomed-stream");
+ firehoseService.flushDueBuffers(clock.instant().plusSeconds(301));
+
+ verify(s3Service, never()).putObject(anyString(), anyString(), any(byte[].class), anyString(), anyMap());
+ }
+
+ @Test
+ void timeBasedFlushDeliversEachBatchOnlyOnce() {
+ firehoseService.createDeliveryStream("idle-stream", null);
+ putRecords("idle-stream", 2);
+
+ Instant afterInterval = clock.instant().plusSeconds(301);
+ firehoseService.flushDueBuffers(afterInterval);
+ firehoseService.flushDueBuffers(afterInterval.plusSeconds(301));
+
+ verify(s3Service).putObject(anyString(), anyString(), any(byte[].class), anyString(), anyMap());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java
index 3e372eaaa7..81be47d0b0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService;
import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService.MetricIdentity;
+import io.github.hectorvent.floci.services.firehose.FirehoseService;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
@@ -83,6 +84,9 @@ class SesEventPublishingV2IntegrationTest {
@Inject
CloudWatchMetricsService metricsService;
+ @Inject
+ FirehoseService firehoseService;
+
@BeforeAll
static void configureRestAssured() {
RestAssuredJsonUtils.configureAwsContentTypes();
@@ -639,22 +643,26 @@ class SesEventPublishingV2IntegrationTest {
@Test
@Order(15)
- void firehose_fiveSends_triggerAutoFlushAndWriteNdjsonToS3() throws Exception {
- for (int i = 0; i < 5; i++) {
- sendEmailToConfigSet(CS_FIREHOSE, "recipient" + i + "@example.com", "fh-evt-" + i);
- }
+ void firehose_sendEventsAreDeliveredAsNdjsonToS3() throws Exception {
+ sendEmailToConfigSet(CS_FIREHOSE, "[email protected]", "fh-evt-0");
+ sendEmailToConfigSet(CS_FIREHOSE, "[email protected]", "fh-evt-1");
+
+ // Small events stay buffered until the stream's size (SizeInMBs) or
+ // interval (IntervalInSeconds) trigger fires; force the flush so the
+ // assertion is deterministic (same mechanism the scheduled flusher uses).
+ firehoseService.flush(FIREHOSE_STREAM);
List<S3Object> objects = s3Service.listObjects(FIREHOSE_BUCKET, FIREHOSE_STREAM + "/", null, 100);
assertEquals(1, objects.size(),
- "expected exactly one flushed S3 object after 5 putRecord calls (DEFAULT_FLUSH_COUNT)");
+ "expected exactly one flushed S3 object containing the buffered events");
S3Object obj = s3Service.getObject(FIREHOSE_BUCKET, objects.get(0).getKey());
String body = new String(obj.getData(), StandardCharsets.UTF_8);
String[] lines = body.split("\\R");
- assertEquals(5, lines.length, "flushed object should contain 5 NDJSON records");
+ assertEquals(2, lines.length, "flushed object should contain one NDJSON record per send");
- for (int i = 0; i < 5; i++) {
- JsonNode event = MAPPER.readTree(lines[i]);
+ for (String line : lines) {
+ JsonNode event = MAPPER.readTree(line);
assertEquals("Send", event.path("eventType").asText());
assertEquals(SENDER, event.path("mail").path("source").asText());
assertEquals(CS_FIREHOSE,
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index 698ded0969..1082e9202a 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -177,6 +177,8 @@ floci:
enabled: true
firehose:
enabled: true
+ tick-interval-seconds: 10
+ flush-record-count: 0
kms:
enabled: true
cognito: