Light, fluffy, and always free - The AWS Local Emulator alternative
feat(firehose): honor BufferingHints with time and size based flush (#2314)
Mihai Blidarubfda938f7223
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.javaindex a249444f6c..ee730a3bd8 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java@@ -9,6 +9,9 @@ import software.amazon.awssdk.services.firehose.model.PutRecordRequest;import software.amazon.awssdk.services.firehose.model.Record;import software.amazon.awssdk.services.glue.GlueClient;import software.amazon.awssdk.services.glue.model.*;+import software.amazon.awssdk.services.s3.S3Client;+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;+import software.amazon.awssdk.services.s3.model.NoSuchBucketException;import java.nio.charset.StandardCharsets;import java.time.Duration;@@ -26,6 +29,7 @@ class DataLakeTest {private static AthenaClient athena;private static GlueClient glue;private static FirehoseClient firehose;+ private static S3Client s3;private static final String DB_NAME = TestFixtures.uniqueName("test_db");private static final String TABLE_NAME = "orders";@@ -36,6 +40,7 @@ class DataLakeTest {athena = TestFixtures.athenaClient();glue = TestFixtures.glueClient();firehose = TestFixtures.firehoseClient();+ s3 = TestFixtures.s3Client();}@Test@@ -76,6 +81,12 @@ class DataLakeTest {.bucketARN("arn:aws:s3:::floci-firehose-results").roleARN("arn:aws:iam::000000000000:role/datalake-firehose-role").prefix(STREAM_NAME + "/")+ // Floci does not enforce AWS's 60s minimum interval, which keeps+ // the delivery wait short (would need 60+ against real AWS).+ .bufferingHints(software.amazon.awssdk.services.firehose.model.BufferingHints.builder()+ .sizeInMBs(1)+ .intervalInSeconds(5)+ .build()).build()).build());}@@ -92,6 +103,28 @@ class DataLakeTest {.build());}+ // Small records stay buffered until the stream's IntervalInSeconds+ // elapses; wait for Firehose to deliver before querying, the same way a+ // real AWS client would.+ long deadline = System.currentTimeMillis() + 30_000;+ boolean delivered = false;+ while (!delivered && System.currentTimeMillis() < deadline) {+ try {+ delivered = !s3.listObjectsV2(ListObjectsV2Request.builder()+ .bucket("floci-firehose-results")+ .prefix(STREAM_NAME + "/")+ .build()).contents().isEmpty();+ } catch (NoSuchBucketException e) {+ // The bucket itself is only created on the first delivery.+ }+ if (!delivered) {+ Thread.sleep(2_000);+ }+ }+ assertThat(delivered)+ .as("Firehose should deliver the buffered records within IntervalInSeconds")+ .isTrue();+// Athena QueryStartQueryExecutionResponse startResp = athena.startQueryExecution(StartQueryExecutionRequest.builder().queryString("SELECT sum(amount) as total FROM " + TABLE_NAME)diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.javaindex 6334012bde..07130e0ca6 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.java@@ -56,6 +56,7 @@ import software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest;import software.amazon.awssdk.services.sqs.model.ReceiveMessageResponse;import software.amazon.awssdk.services.firehose.FirehoseClient;+import software.amazon.awssdk.services.firehose.model.BufferingHints;import software.amazon.awssdk.services.firehose.model.CreateDeliveryStreamRequest;import software.amazon.awssdk.services.firehose.model.DeleteDeliveryStreamRequest;import software.amazon.awssdk.services.firehose.model.S3DestinationConfiguration;@@ -199,6 +200,12 @@ class SesEventPublishingTest {.bucketARN("arn:aws:s3:::" + firehoseBucket).roleARN("arn:aws:iam::000000000000:role/sdk-evt-fh-role").prefix(firehoseStreamName + "/")+ // Floci does not enforce AWS's 60s minimum interval, which keeps+ // the delivery wait short (would need 60+ against real AWS).+ .bufferingHints(BufferingHints.builder()+ .sizeInMBs(1)+ .intervalInSeconds(5)+ .build()).build()).build()).deliveryStreamARN();@@ -478,9 +485,9 @@ class SesEventPublishingTest {@Test@Order(5)- @DisplayName("Firehose destination: 5 sends trigger auto-flush to S3 as NDJSON")- void firehoseDestination_fiveSends_triggerAutoFlushToS3() throws Exception {- for (int i = 0; i < 5; i++) {+ @DisplayName("Firehose destination: send events are delivered to S3 as NDJSON within the buffering interval")+ void firehoseDestination_sendEvents_deliveredWithinBufferingInterval() throws Exception {+ for (int i = 0; i < 2; i++) {ses.sendEmail(SendEmailRequest.builder().fromEmailAddress(identity).destination(Destination.builder()@@ -498,12 +505,22 @@ class SesEventPublishingTest {.build());}- ListObjectsV2Response listed = s3.listObjectsV2(ListObjectsV2Request.builder()- .bucket(firehoseBucket)- .prefix(firehoseStreamName + "/")- .build());+ // Two small events stay well below SizeInMBs, so delivery happens on the+ // stream's IntervalInSeconds plus the emulator's flush tick.+ ListObjectsV2Response listed = null;+ long deadline = System.currentTimeMillis() + 30_000;+ while (System.currentTimeMillis() < deadline) {+ listed = s3.listObjectsV2(ListObjectsV2Request.builder()+ .bucket(firehoseBucket)+ .prefix(firehoseStreamName + "/")+ .build());+ if (!listed.contents().isEmpty()) {+ break;+ }+ Thread.sleep(2_000);+ }assertThat(listed.contents())- .as("Firehose should have flushed exactly one S3 object after 5 records")+ .as("Firehose should deliver the buffered events within IntervalInSeconds").hasSize(1);S3Object obj = listed.contents().get(0);@@ -511,7 +528,7 @@ class SesEventPublishingTest {.bucket(firehoseBucket).key(obj.key()).build()).readAllBytes(), StandardCharsets.UTF_8);String[] lines = body.split("\\R");- assertThat(lines).hasSize(5);+ assertThat(lines).hasSize(2);for (String line : lines) {JsonNode event = MAPPER.readTree(line);assertThat(event.path("eventType").asText()).isEqualTo("Send");diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.mdindex 7762a50b63..f4ef6860fe 100644--- a/docs/configuration/environment-variables.md+++ b/docs/configuration/environment-variables.md@@ -232,6 +232,8 @@ See [Initialization Hooks](./initialization-hooks.md) for lifecycle phases and s| Variable | Default | Description ||---|---|---|| `FLOCI_SERVICES_FIREHOSE_ENABLED` | `true` | Enable the Kinesis Data Firehose service |+| `FLOCI_SERVICES_FIREHOSE_TICK_INTERVAL_SECONDS` | `10` | How often (seconds) the buffer flusher checks for streams whose `BufferingHints.IntervalInSeconds` has elapsed |+| `FLOCI_SERVICES_FIREHOSE_FLUSH_RECORD_COUNT` | `0` | Emulator-only: flush after this many buffered records (`0` = disabled, AWS-faithful; `1` = LocalStack-style record-at-a-time delivery) |### EventBridgediff --git a/docs/services/firehose.md b/docs/services/firehose.mdindex 5d96befbb5..d3ceeb1661 100644--- a/docs/services/firehose.md+++ b/docs/services/firehose.md@@ -25,7 +25,13 @@ Floci emulates Amazon Data Firehose for streaming data ingestion and delivery to## How it works1. **Buffering**: Incoming records are buffered in memory.-2. **Automatic Flush**: Floci automatically flushes the buffer to S3 after every 5 records for immediate local feedback.+2. **Automatic Flush**: A buffer is delivered to S3 when either trigger of the stream's `BufferingHints` fires first, mirroring AWS's buffered delivery:+ - the buffered records reach `SizeInMBs` (default 5 MiB), or+ - `IntervalInSeconds` (default 300 s) has elapsed since the first buffered record. A background flusher checks this every `floci.services.firehose.tick-interval-seconds` (default 10 s).++ An emulator-only record-count trigger is also available for local dev: set `floci.services.firehose.flush-record-count` (env: `FLOCI_SERVICES_FIREHOSE_FLUSH_RECORD_COUNT`) to flush after that many buffered records — `1` gives LocalStack-style record-at-a-time delivery. Disabled by default (`0`) so delivery timing matches real AWS.++ Buffers are also flushed on shutdown, so pending records are not lost. Deleting a delivery stream discards its pending records without flushing, matching real AWS.3. **Format**: Records are flushed as raw NDJSON (newline-delimited JSON) to the bucket configured in the S3 destination (`floci-firehose-results` if the stream has no destination configuration).## S3 object keysdiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 6d45d36839..8b4580ea04 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -1083,6 +1083,22 @@ public interface EmulatorConfig {interface FirehoseServiceConfig {@WithDefault("true")boolean enabled();++ /**+ * How often the buffer flusher checks for streams whose buffering+ * interval (BufferingHints.IntervalInSeconds) has elapsed.+ */+ @WithDefault("10")+ long tickIntervalSeconds();++ /**+ * Emulator-only volume trigger: number of buffered records that forces+ * an immediate flush, complementing the stream's BufferingHints.+ * Disabled by default (0) so out-of-the-box delivery matches real AWS;+ * set to 1 for LocalStack-style record-at-a-time delivery in local dev.+ */+ @WithDefault("0")+ int flushRecordCount();}interface KmsServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.javaindex 0577c0353d..57b3cd2149 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.firehose;import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;@@ -11,37 +12,114 @@ import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescriptimport io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;import io.github.hectorvent.floci.services.firehose.model.Record;import io.github.hectorvent.floci.services.s3.S3Service;+import io.quarkus.runtime.ShutdownDelayInitiatedEvent;+import io.quarkus.runtime.StartupEvent;import jakarta.enterprise.context.ApplicationScoped;+import jakarta.enterprise.event.Observes;import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.nio.charset.StandardCharsets;import java.time.Clock;+import java.time.Instant;import java.time.ZoneId;import java.util.*;import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.Executors;+import java.util.concurrent.ScheduledExecutorService;+import java.util.concurrent.TimeUnit;@ApplicationScopedpublic class FirehoseService {private static final Logger LOG = Logger.getLogger(FirehoseService.class);private static final String DEFAULT_BUCKET = "floci-firehose-results";- private static final int DEFAULT_FLUSH_COUNT = 5;+ private static final int DEFAULT_BUFFERING_INTERVAL_SECONDS = 300;+ private static final int DEFAULT_BUFFERING_SIZE_MBS = 5;private final StorageBackend<String, DeliveryStreamDescription> streamStore;private final Map<String, List<byte[]>> buffers = new ConcurrentHashMap<>();+ private final Map<String, Instant> bufferSince = new ConcurrentHashMap<>();private final S3Service s3Service;private final RegionResolver regionResolver;private final Clock clock;+ private final long tickIntervalSeconds;+ private final int flushRecordCount;+ private final boolean flusherEnabled;+ private final ScheduledExecutorService flushExecutor;@Injectpublic FirehoseService(StorageFactory storageFactory, S3Service s3Service, RegionResolver regionResolver,- Clock clock) {+ Clock clock, EmulatorConfig config) {this.streamStore = storageFactory.create("firehose", "streams.json",new TypeReference<Map<String, DeliveryStreamDescription>>() {});this.s3Service = s3Service;this.regionResolver = regionResolver;this.clock = clock;+ this.tickIntervalSeconds = Math.max(1, config.services().firehose().tickIntervalSeconds());+ this.flushRecordCount = Math.max(0, config.services().firehose().flushRecordCount());+ this.flusherEnabled = config.services().firehose().enabled();+ this.flushExecutor = Executors.newSingleThreadScheduledExecutor(r -> {+ Thread t = new Thread(r, "firehose-buffer-flusher");+ t.setDaemon(true);+ return t;+ });+ }++ void onStart(@Observes StartupEvent ignored) {+ if (!flusherEnabled) {+ LOG.info("Firehose buffer flusher disabled by configuration");+ return;+ }+ flushExecutor.scheduleAtFixedRate(this::tickSafely, tickIntervalSeconds, tickIntervalSeconds, TimeUnit.SECONDS);+ LOG.infov("Firehose buffer flusher started (tick every {0}s)", tickIntervalSeconds);+ }++ // ShutdownDelayInitiatedEvent fires before every ShutdownEvent observer, so this+ // drain lands the pending records in S3 while EmulatorLifecycle.onStop can still+ // persist them to disk via storageFactory.flushAll().+ void onPreShutdown(@Observes ShutdownDelayInitiatedEvent ignored) {+ flushExecutor.shutdownNow();+ buffers.keySet().forEach(this::flush);+ }++ void tickSafely() {+ try {+ flushDueBuffers(clock.instant());+ } catch (Throwable t) {+ LOG.warnv("Firehose buffer flush tick failed: {0}", t.getMessage());+ }+ }++ void flushDueBuffers(Instant now) {+ for (Map.Entry<String, List<byte[]>> entry : buffers.entrySet()) {+ if (entry.getValue().isEmpty()) {+ continue;+ }+ String streamName = entry.getKey();+ try {+ DeliveryStreamDescription stream = describeDeliveryStream(streamName);+ Instant since = bufferSince.putIfAbsent(streamName, now);+ if (since == null) {+ since = now;+ }+ if (!now.isBefore(since.plusSeconds(bufferingIntervalSeconds(stream)))) {+ flush(streamName, stream);+ }+ } catch (Exception e) {+ LOG.warnv("Firehose buffer flush failed for stream {0}: {1}", streamName, e.getMessage());+ }+ }+ }++ private static int bufferingIntervalSeconds(DeliveryStreamDescription stream) {+ S3Destination s3 = stream.s3Destination();+ // describeDeliveryStream already applied defaults, so hints only miss+ // when the stream has no S3 destination at all (default-bucket delivery).+ if (s3 == null || s3.getBufferingHints() == null || s3.getBufferingHints().getIntervalInSeconds() == null) {+ return DEFAULT_BUFFERING_INTERVAL_SECONDS;+ }+ return s3.getBufferingHints().getIntervalInSeconds();}public String createDeliveryStream(String name, S3Destination s3Config) {@@ -219,7 +297,11 @@ public class FirehoseService {public void deleteDeliveryStream(String name) {describeDeliveryStream(name);streamStore.delete(name);+ // Pending records are discarded, not flushed: verified against real AWS+ // (2026-07-13, eu-west-1) — 3 records buffered under a 300s/5MB hint never+ // reached the bucket after DeleteDeliveryStream completed.buffers.remove(name);+ bufferSince.remove(name);LOG.infov("Deleted Firehose delivery stream: {0}", name);}@@ -229,27 +311,41 @@ public class FirehoseService {}public void putRecord(String streamName, Record record) {- DeliveryStreamDescription stream = describeDeliveryStream(streamName);- buffers.computeIfAbsent(streamName, k -> Collections.synchronizedList(new ArrayList<>()))- .add(record.getData());-- if (buffers.get(streamName).size() >= DEFAULT_FLUSH_COUNT) {- flush(streamName, stream);- }+ putRecordBatch(streamName, List.of(record));}public void putRecordBatch(String streamName, List<Record> records) {DeliveryStreamDescription stream = describeDeliveryStream(streamName);List<byte[]> buffer = buffers.computeIfAbsent(streamName, k -> Collections.synchronizedList(new ArrayList<>()));- for (Record r : records) {- buffer.add(r.getData());+ long bufferedBytes = 0;+ int bufferedCount;+ // Records and their buffering-start timestamp move together under the+ // buffer lock so the flusher never sees one without the other.+ synchronized (buffer) {+ for (Record r : records) {+ buffer.add(r.getData());+ }+ bufferSince.putIfAbsent(streamName, clock.instant());+ bufferedCount = buffer.size();+ for (byte[] data : buffer) {+ bufferedBytes += data.length;+ }}- if (buffer.size() >= DEFAULT_FLUSH_COUNT) {+ if ((flushRecordCount > 0 && bufferedCount >= flushRecordCount)+ || bufferedBytes >= bufferingSizeLimitBytes(stream)) {flush(streamName, stream);}}+ private static long bufferingSizeLimitBytes(DeliveryStreamDescription stream) {+ S3Destination s3 = stream.s3Destination();+ int sizeInMBs = (s3 == null || s3.getBufferingHints() == null || s3.getBufferingHints().getSizeInMBs() == null)+ ? DEFAULT_BUFFERING_SIZE_MBS+ : s3.getBufferingHints().getSizeInMBs();+ return sizeInMBs * 1024L * 1024L;+ }+public void flush(String streamName) {streamStore.get(streamName).ifPresent(stream -> flush(streamName, stream));}@@ -264,6 +360,11 @@ public class FirehoseService {synchronized (buffer) {toFlush = new ArrayList<>(buffer);buffer.clear();+ bufferSince.remove(streamName);+ }+ if (toFlush.isEmpty()) {+ // Lost the race against a concurrent flush; nothing left to deliver.+ return;}try {diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 95de580ceb..280c84b52b 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -312,6 +312,8 @@ floci:enabled: truefirehose:enabled: true+ tick-interval-seconds: 10+ flush-record-count: 0kms:enabled: truecognito:diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.javaindex 406471215d..e29d267bab 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.firehose;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.MethodOrderer;import org.junit.jupiter.api.Order;@@ -20,6 +21,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue;@TestMethodOrder(MethodOrderer.OrderAnnotation.class)class FirehoseExtendedS3IntegrationTest {+ @Inject+ FirehoseService firehoseService;+private static final String STREAM_NAME = "test-extended-s3-stream";private static final String LEGACY_STREAM_NAME = "test-legacy-s3-stream";private static final String MINIMAL_STREAM_NAME = "test-minimal-extended-s3-stream";@@ -339,7 +343,7 @@ class FirehoseExtendedS3IntegrationTest {String stream = "delivery-default-stream";String bucket = "extended-s3-delivery-default";createDeliveryStream(stream, bucket, "");- putFiveRecords(stream);+ putFiveRecordsAndFlush(stream);String key = firstDeliveredKey(bucket, "");assertTrue(key.matches(TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);@@ -351,7 +355,7 @@ class FirehoseExtendedS3IntegrationTest {String stream = "delivery-static-stream";String bucket = "extended-s3-delivery-static";createDeliveryStream(stream, bucket, ", \"Prefix\": \"events/data/\"");- putFiveRecords(stream);+ putFiveRecordsAndFlush(stream);String key = firstDeliveredKey(bucket, "events/data/");assertTrue(key.matches("events/data/" + TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);@@ -363,7 +367,7 @@ class FirehoseExtendedS3IntegrationTest {String stream = "delivery-noslash-stream";String bucket = "extended-s3-delivery-noslash";createDeliveryStream(stream, bucket, ", \"Prefix\": \"legacy\"");- putFiveRecords(stream);+ putFiveRecordsAndFlush(stream);String key = firstDeliveredKey(bucket, "legacy");assertTrue(key.matches("legacy" + TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);@@ -375,7 +379,7 @@ class FirehoseExtendedS3IntegrationTest {String stream = "delivery-expr-stream";String bucket = "extended-s3-delivery-expr";createDeliveryStream(stream, bucket, ", \"Prefix\": \"data/!{timestamp:yyyy/MM/dd}/\"");- putFiveRecords(stream);+ putFiveRecordsAndFlush(stream);String key = firstDeliveredKey(bucket, "data/");assertTrue(key.matches("data/\\d{4}/\\d{2}/\\d{2}/" + stream + SUFFIX_REGEX), key);@@ -387,7 +391,7 @@ class FirehoseExtendedS3IntegrationTest {String stream = "delivery-rand-stream";String bucket = "extended-s3-delivery-rand";createDeliveryStream(stream, bucket, ", \"Prefix\": \"rand/!{firehose:random-string}/\"");- putFiveRecords(stream);+ putFiveRecordsAndFlush(stream);String key = firstDeliveredKey(bucket, "rand/");assertTrue(key.matches("rand/[A-Za-z0-9]{11}/" + stream + SUFFIX_REGEX), key);@@ -412,7 +416,7 @@ class FirehoseExtendedS3IntegrationTest {.body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CustomTimeZone",equalTo("Europe/Madrid"));- putFiveRecords(stream);+ putFiveRecordsAndFlush(stream);String key = firstDeliveredKey(bucket, "events/");assertTrue(key.matches("events/" + TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);}@@ -436,8 +440,12 @@ class FirehoseExtendedS3IntegrationTest {.statusCode(200);}- /** Five records reach DEFAULT_FLUSH_COUNT, so the batch triggers the automatic flush. */- private void putFiveRecords(String streamName) {+ /**+ * Small records stay buffered until the size (SizeInMBs) or interval+ * (IntervalInSeconds) trigger fires; force the flush so the delivery+ * assertions are deterministic (same mechanism the scheduled flusher uses).+ */+ private void putFiveRecordsAndFlush(String streamName) {String data = Base64.getEncoder().encodeToString("{\"id\": 1}".getBytes(StandardCharsets.UTF_8));given().contentType(CONTENT_TYPE)@@ -455,6 +463,7 @@ class FirehoseExtendedS3IntegrationTest {.then().statusCode(200).body("FailedPutCount", equalTo(0));+ firehoseService.flush(streamName);}private String firstDeliveredKey(String bucket, String listPrefix) {diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushIntegrationTest.javanew file mode 100644index 0000000000..267c57fdc6--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushIntegrationTest.java@@ -0,0 +1,149 @@+package io.github.hectorvent.floci.services.firehose;++import io.github.hectorvent.floci.testing.MutableClock;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;+import jakarta.inject.Inject;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.nio.charset.StandardCharsets;+import java.time.Duration;+import java.util.Base64;+import java.util.Map;+import java.util.regex.Matcher;+import java.util.regex.Pattern;++import static io.restassured.RestAssured.given;+import static org.awaitility.Awaitility.await;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.notNullValue;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * Wire-level coverage of the AWS-faithful BufferingHints flush triggers:+ * volume ({@code SizeInMBs}) and time ({@code IntervalInSeconds}), delivered+ * by the background flusher without any explicit flush call.+ *+ * The emulator-only record-count trigger is covered by+ * {@link FirehoseFlushRecordCountIntegrationTest}: it needs a different+ * config override, and a Quarkus test profile applies to the whole class.+ */+@QuarkusTest+@TestProfile(FirehoseFlushIntegrationTest.FastTickProfile.class)+class FirehoseFlushIntegrationTest {++ public static class FastTickProfile implements QuarkusTestProfile {+ @Override+ public Map<String, String> getConfigOverrides() {+ // 1s flush tick so the interval test waits seconds, not the 10s default.+ return Map.of("floci.services.firehose.tick-interval-seconds", "1");+ }+ }++ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";+ private static final String TARGET_PREFIX = "Firehose_20150804.";+ private static final Pattern KEY_PATTERN = Pattern.compile("<Key>([^<]+)</Key>");++ /** The global test-scope Clock alternative, frozen at 2026-01-01 unless advanced. */+ @Inject+ MutableClock clock;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void sizeTriggerDeliversWhenBufferedBytesReachSizeInMBs() {+ String stream = "flush-size-stream";+ String bucket = "flush-size-archive";+ createDeliveryStream(stream, bucket, 1, 300);++ String halfMiB = Base64.getEncoder()+ .encodeToString("A".repeat(512 * 1024).getBytes(StandardCharsets.UTF_8));+ putRecord(stream, halfMiB);+ putRecord(stream, halfMiB); // crosses the 1 MiB threshold: flushes inline, no waiting++ String body = given()+ .when()+ .get("/" + bucket + "/" + firstDeliveredKey(bucket))+ .then()+ .statusCode(200)+ .extract().asString();+ // Both records, each followed by the newline the flush appends.+ assertEquals(2 * 512 * 1024 + 2, body.length());+ assertTrue(body.startsWith("A"), "delivered body should contain the record payload");+ }++ @Test+ void intervalTriggerDeliversAfterIntervalInSecondsElapses() {+ String stream = "flush-interval-stream";+ String bucket = "flush-interval-archive";+ createDeliveryStream(stream, bucket, 5, 60);++ putRecord(stream, Base64.getEncoder()+ .encodeToString("{\"n\":42}".getBytes(StandardCharsets.UTF_8)));++ // The test-scope Clock is frozen, so the interval can only elapse by+ // advancing it; the real 1s tick then finds the buffer due.+ clock.advance(Duration.ofSeconds(61));++ await().atMost(Duration.ofSeconds(15)).untilAsserted(() ->+ given()+ .when()+ .get("/" + bucket + "/" + firstDeliveredKey(bucket))+ .then()+ .statusCode(200)+ .body(equalTo("{\"n\":42}\n")));+ }++ private void createDeliveryStream(String streamName, String bucket, int sizeInMBs, int intervalInSeconds) {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "DeliveryStreamType": "DirectPut",+ "ExtendedS3DestinationConfiguration": {+ "RoleARN": "arn:aws:iam::000000000000:role/firehose-delivery-role",+ "BucketARN": "arn:aws:s3:::%s",+ "BufferingHints": {"SizeInMBs": %d, "IntervalInSeconds": %d}+ }+ }+ """.formatted(streamName, bucket, sizeInMBs, intervalInSeconds))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DeliveryStreamARN", notNullValue());+ }++ private void putRecord(String streamName, String base64Data) {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "PutRecord")+ .body("{ \"DeliveryStreamName\": \"" + streamName + "\", \"Record\": {\"Data\": \"" + base64Data + "\"} }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("RecordId", notNullValue());+ }++ private String firstDeliveredKey(String bucket) {+ String listing = given()+ .when()+ .get("/" + bucket)+ .then()+ .statusCode(200)+ .extract().asString();+ Matcher key = KEY_PATTERN.matcher(listing);+ assertTrue(key.find(), "expected a delivered object in " + bucket + ", got: " + listing);+ return key.group(1);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushRecordCountIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushRecordCountIntegrationTest.javanew file mode 100644index 0000000000..cb080c03a7--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseFlushRecordCountIntegrationTest.java@@ -0,0 +1,109 @@+package io.github.hectorvent.floci.services.firehose;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import java.util.Map;+import java.util.regex.Matcher;+import java.util.regex.Pattern;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.notNullValue;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * Verifies {@code floci.services.firehose.flush-record-count=1} delivers+ * every record to the S3 destination immediately.+ *+ * Kept separate from {@link FirehoseFlushIntegrationTest} (size and interval+ * triggers) because this class needs a different config override and a+ * Quarkus test profile applies to the whole class.+ */+@QuarkusTest+@TestProfile(FirehoseFlushRecordCountIntegrationTest.FlushEveryRecordProfile.class)+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class FirehoseFlushRecordCountIntegrationTest {++ public static class FlushEveryRecordProfile implements QuarkusTestProfile {+ @Override+ public Map<String, String> getConfigOverrides() {+ return Map.of("floci.services.firehose.flush-record-count", "1");+ }+ }++ private static final String STREAM_NAME = "test-flush-count-stream";+ private static final String BUCKET = "flush-count-archive";+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";+ private static final String TARGET_PREFIX = "Firehose_20150804.";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ @Order(1)+ void createDeliveryStream() {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "DeliveryStreamType": "DirectPut",+ "ExtendedS3DestinationConfiguration": {+ "RoleARN": "arn:aws:iam::000000000000:role/firehose-delivery-role",+ "BucketARN": "arn:aws:s3:::%s",+ "Prefix": "single/"+ }+ }+ """.formatted(STREAM_NAME, BUCKET))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DeliveryStreamARN", notNullValue());+ }++ @Test+ @Order(2)+ void singleRecordIsDeliveredImmediately() {+ // {"n":1} — base64: eyJuIjoxfQ==+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "PutRecord")+ .body("{ \"DeliveryStreamName\": \"" + STREAM_NAME + "\", \"Record\": {\"Data\": \"eyJuIjoxfQ==\"} }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("RecordId", notNullValue());++ // The record must be visible in the destination bucket without any+ // further puts (path-style S3 ListObjects on the same edge port).+ String listing = given()+ .when()+ .get("/" + BUCKET)+ .then()+ .statusCode(200)+ .extract().asString();++ Matcher key = Pattern.compile("<Key>(single/[^<]+)</Key>").matcher(listing);+ assertTrue(key.find(), "expected a delivered object under single/, got: " + listing);++ given()+ .when()+ .get("/" + BUCKET + "/" + key.group(1))+ .then()+ .statusCode(200)+ .body(equalTo("{\"n\":1}\n"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.javaindex b83fbeefe2..13357d3e89 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java@@ -1,7 +1,7 @@package io.github.hectorvent.floci.services.firehose;+import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.RegionResolver;-import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;@@ -15,12 +15,16 @@ import org.mockito.ArgumentCaptor;import org.mockito.Mockito;import java.nio.charset.StandardCharsets;+import java.time.Instant;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyMap;import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;@@ -29,25 +33,46 @@ class FirehoseServiceTest {private static final String UUID_REGEX = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";private FirehoseService firehoseService;+ private StorageFactory storageFactory;private S3Service s3Service;+ private MutableClock clock;@BeforeEachvoid setUp() {- StorageFactory storageFactory = Mockito.mock(StorageFactory.class);+ storageFactory = Mockito.mock(StorageFactory.class);when(storageFactory.create(anyString(), anyString(), any())).thenReturn(AccountAwareStorageBackend.inMemory("000000000000"));s3Service = Mockito.mock(S3Service.class);- firehoseService = new FirehoseService(storageFactory, s3Service,- new RegionResolver("us-east-1", "000000000000"), new MutableClock());+ clock = new MutableClock();+ firehoseService = newService(0);}- private void putRecordsUntilFlush(String streamName) {- for (int i = 0; i < 5; i++) {- Record record = new Record(("{\"n\":" + i + "}").getBytes(StandardCharsets.UTF_8));- firehoseService.putRecord(streamName, record);+ private FirehoseService newService(int flushRecordCount) {+ EmulatorConfig.FirehoseServiceConfig firehoseCfg = mock(EmulatorConfig.FirehoseServiceConfig.class);+ when(firehoseCfg.enabled()).thenReturn(true);+ when(firehoseCfg.tickIntervalSeconds()).thenReturn(10L);+ when(firehoseCfg.flushRecordCount()).thenReturn(flushRecordCount);+ EmulatorConfig.ServicesConfig servicesCfg = mock(EmulatorConfig.ServicesConfig.class);+ when(servicesCfg.firehose()).thenReturn(firehoseCfg);+ EmulatorConfig config = mock(EmulatorConfig.class);+ when(config.services()).thenReturn(servicesCfg);++ return new FirehoseService(storageFactory, s3Service,+ new RegionResolver("us-east-1", "000000000000"), clock, config);+ }++ private void putRecords(String streamName, int count) {+ for (int i = 0; i < count; i++) {+ firehoseService.putRecord(streamName, new Record(("{\"n\":" + i + "}").getBytes(StandardCharsets.UTF_8)));}}+ /** Puts a few small records and forces delivery, the way the interval trigger eventually would. */+ private void putRecordsAndFlush(String streamName) {+ putRecords(streamName, 5);+ firehoseService.flush(streamName);+ }+private String deliveredKey(String expectedBucket) {ArgumentCaptor<String> bucket = ArgumentCaptor.forClass(String.class);ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);@@ -59,7 +84,7 @@ class FirehoseServiceTest {@Testvoid deliversToDefaultBucketWithAwsShapedKey() {firehoseService.createDeliveryStream("my-stream", null);- putRecordsUntilFlush("my-stream");+ putRecordsAndFlush("my-stream");String key = deliveredKey("floci-firehose-results");assertTrue(key.matches("2026/01/01/00/my-stream-1-2026-01-01-00-00-00-" + UUID_REGEX), key);@@ -71,7 +96,7 @@ class FirehoseServiceTest {s3.setBucketArn("arn:aws:s3:::custom-bucket");s3.setPrefix("events/data/");firehoseService.createDeliveryStream("my-stream", s3);- putRecordsUntilFlush("my-stream");+ putRecordsAndFlush("my-stream");String key = deliveredKey("custom-bucket");assertTrue(key.matches("events/data/2026/01/01/00/my-stream-1-2026-01-01-00-00-00-" + UUID_REGEX), key);@@ -83,7 +108,7 @@ class FirehoseServiceTest {s3.setBucketArn("arn:aws:s3:::custom-bucket");s3.setCustomTimeZone("Europe/Madrid");firehoseService.createDeliveryStream("my-stream", s3);- putRecordsUntilFlush("my-stream");+ putRecordsAndFlush("my-stream");String key = deliveredKey("custom-bucket");assertTrue(key.matches("2026/01/01/01/my-stream-1-2026-01-01-01-00-00-" + UUID_REGEX), key);@@ -110,8 +135,129 @@ class FirehoseServiceTest {assertEquals("Asia/Tokyo",firehoseService.describeDeliveryStream("my-stream").s3Destination().getCustomTimeZone());- putRecordsUntilFlush("my-stream");+ putRecordsAndFlush("my-stream");String key = deliveredKey("custom-bucket");assertTrue(key.matches("events/2026/01/01/09/my-stream-3-2026-01-01-09-00-00-" + UUID_REGEX), key);}++ @Test+ void timeBasedFlushKeepsBufferWhileDefaultIntervalHasNotElapsed() {+ firehoseService.createDeliveryStream("idle-stream", null);+ putRecords("idle-stream", 2);++ firehoseService.flushDueBuffers(clock.instant().plusSeconds(299));++ verify(s3Service, never()).putObject(anyString(), anyString(), any(byte[].class), anyString(), anyMap());+ }++ @Test+ void timeBasedFlushDeliversBufferedRecordsAfterDefaultInterval() {+ firehoseService.createDeliveryStream("idle-stream", null);+ putRecords("idle-stream", 2);++ firehoseService.flushDueBuffers(clock.instant().plusSeconds(300));++ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);+ verify(s3Service).putObject(eq("floci-firehose-results"), anyString(), body.capture(), anyString(), anyMap());+ assertEquals("{\"n\":0}\n{\"n\":1}\n", new String(body.getValue(), StandardCharsets.UTF_8));+ }++ @Test+ void timeBasedFlushHonorsStreamBufferingIntervalHint() {+ S3Destination s3 = new S3Destination();+ s3.setBucketArn("arn:aws:s3:::custom-bucket");+ DeliveryStreamDescription.BufferingHints hints = new DeliveryStreamDescription.BufferingHints();+ hints.setSizeInMBs(5);+ hints.setIntervalInSeconds(60);+ s3.setBufferingHints(hints);+ firehoseService.createDeliveryStream("hinted-stream", s3);+ putRecords("hinted-stream", 1);++ firehoseService.flushDueBuffers(clock.instant().plusSeconds(59));+ verify(s3Service, never()).putObject(anyString(), anyString(), any(byte[].class), anyString(), anyMap());++ firehoseService.flushDueBuffers(clock.instant().plusSeconds(60));+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);+ verify(s3Service).putObject(eq("custom-bucket"), anyString(), body.capture(), anyString(), anyMap());+ assertEquals("{\"n\":0}\n", new String(body.getValue(), StandardCharsets.UTF_8));+ }++ /** Matches real AWS: the volume trigger is bytes vs SizeInMBs, never a record count. */+ @Test+ void smallRecordsDoNotTriggerSizeBasedFlushRegardlessOfCount() {+ firehoseService.createDeliveryStream("trickle-stream", null);+ putRecords("trickle-stream", 20);++ verify(s3Service, never()).putObject(anyString(), anyString(), any(byte[].class), anyString(), anyMap());+ }++ @Test+ void sizeBasedFlushDeliversWhenBufferedBytesReachSizeHint() {+ S3Destination s3 = new S3Destination();+ s3.setBucketArn("arn:aws:s3:::custom-bucket");+ DeliveryStreamDescription.BufferingHints hints = new DeliveryStreamDescription.BufferingHints();+ hints.setSizeInMBs(1);+ hints.setIntervalInSeconds(300);+ s3.setBufferingHints(hints);+ firehoseService.createDeliveryStream("bulky-stream", s3);++ firehoseService.putRecord("bulky-stream", new Record(new byte[512 * 1024]));+ verify(s3Service, never()).putObject(anyString(), anyString(), any(byte[].class), anyString(), anyMap());++ firehoseService.putRecord("bulky-stream", new Record(new byte[512 * 1024]));+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);+ verify(s3Service).putObject(eq("custom-bucket"), anyString(), body.capture(), anyString(), anyMap());+ // Both 512 KiB records, each followed by the newline the flush appends.+ assertEquals(2 * 512 * 1024 + 2, body.getValue().length);+ }++ /** Emulator-only opt-in: flush-record-count=1 restores LocalStack-style record-at-a-time delivery. */+ @Test+ void flushRecordCountOfOneDeliversEachRecordImmediately() {+ firehoseService = newService(1);+ firehoseService.createDeliveryStream("eager-stream", null);+ firehoseService.putRecord("eager-stream", new Record("{\"n\":0}".getBytes(StandardCharsets.UTF_8)));++ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);+ verify(s3Service).putObject(eq("floci-firehose-results"), anyString(), body.capture(), anyString(), anyMap());+ assertEquals("{\"n\":0}\n", new String(body.getValue(), StandardCharsets.UTF_8));+ }++ @Test+ void flushRecordCountThresholdDeliversTheWholeBuffer() {+ firehoseService = newService(3);+ firehoseService.createDeliveryStream("counted-stream", null);+ putRecords("counted-stream", 2);+ verify(s3Service, never()).putObject(anyString(), anyString(), any(byte[].class), anyString(), anyMap());++ firehoseService.putRecord("counted-stream", new Record("{\"n\":2}".getBytes(StandardCharsets.UTF_8)));++ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);+ verify(s3Service).putObject(eq("floci-firehose-results"), anyString(), body.capture(), anyString(), anyMap());+ assertEquals("{\"n\":0}\n{\"n\":1}\n{\"n\":2}\n", new String(body.getValue(), StandardCharsets.UTF_8));+ }++ /** Matches real AWS: DeleteDeliveryStream discards undelivered records instead of flushing them. */+ @Test+ void deleteDeliveryStreamDiscardsBufferedRecords() {+ firehoseService.createDeliveryStream("doomed-stream", null);+ putRecords("doomed-stream", 2);++ firehoseService.deleteDeliveryStream("doomed-stream");+ firehoseService.flushDueBuffers(clock.instant().plusSeconds(301));++ verify(s3Service, never()).putObject(anyString(), anyString(), any(byte[].class), anyString(), anyMap());+ }++ @Test+ void timeBasedFlushDeliversEachBatchOnlyOnce() {+ firehoseService.createDeliveryStream("idle-stream", null);+ putRecords("idle-stream", 2);++ Instant afterInterval = clock.instant().plusSeconds(301);+ firehoseService.flushDueBuffers(afterInterval);+ firehoseService.flushDueBuffers(afterInterval.plusSeconds(301));++ verify(s3Service).putObject(anyString(), anyString(), any(byte[].class), anyString(), anyMap());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.javaindex 3e372eaaa7..81be47d0b0 100644--- a/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService;import io.github.hectorvent.floci.services.cloudwatch.metrics.CloudWatchMetricsService.MetricIdentity;+import io.github.hectorvent.floci.services.firehose.FirehoseService;import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.s3.model.S3Object;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;@@ -83,6 +84,9 @@ class SesEventPublishingV2IntegrationTest {@InjectCloudWatchMetricsService metricsService;+ @Inject+ FirehoseService firehoseService;+@BeforeAllstatic void configureRestAssured() {RestAssuredJsonUtils.configureAwsContentTypes();@@ -639,22 +643,26 @@ class SesEventPublishingV2IntegrationTest {@Test@Order(15)- void firehose_fiveSends_triggerAutoFlushAndWriteNdjsonToS3() throws Exception {- for (int i = 0; i < 5; i++) {- sendEmailToConfigSet(CS_FIREHOSE, "recipient" + i + "@example.com", "fh-evt-" + i);- }+ void firehose_sendEventsAreDeliveredAsNdjsonToS3() throws Exception {+ sendEmailToConfigSet(CS_FIREHOSE, "[email protected]", "fh-evt-0");+ sendEmailToConfigSet(CS_FIREHOSE, "[email protected]", "fh-evt-1");++ // Small events stay buffered until the stream's size (SizeInMBs) or+ // interval (IntervalInSeconds) trigger fires; force the flush so the+ // assertion is deterministic (same mechanism the scheduled flusher uses).+ firehoseService.flush(FIREHOSE_STREAM);List<S3Object> objects = s3Service.listObjects(FIREHOSE_BUCKET, FIREHOSE_STREAM + "/", null, 100);assertEquals(1, objects.size(),- "expected exactly one flushed S3 object after 5 putRecord calls (DEFAULT_FLUSH_COUNT)");+ "expected exactly one flushed S3 object containing the buffered events");S3Object obj = s3Service.getObject(FIREHOSE_BUCKET, objects.get(0).getKey());String body = new String(obj.getData(), StandardCharsets.UTF_8);String[] lines = body.split("\\R");- assertEquals(5, lines.length, "flushed object should contain 5 NDJSON records");+ assertEquals(2, lines.length, "flushed object should contain one NDJSON record per send");- for (int i = 0; i < 5; i++) {- JsonNode event = MAPPER.readTree(lines[i]);+ for (String line : lines) {+ JsonNode event = MAPPER.readTree(line);assertEquals("Send", event.path("eventType").asText());assertEquals(SENDER, event.path("mail").path("source").asText());assertEquals(CS_FIREHOSE,diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 698ded0969..1082e9202a 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -177,6 +177,8 @@ floci:enabled: truefirehose:enabled: true+ tick-interval-seconds: 10+ flush-record-count: 0kms:enabled: truecognito: