Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(firehose): build S3 object keys with AWS prefix expressions and name suffix (#1858)

Mihai Blidarua79e04e4ecd9
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java
index b53e754d8e..a249444f6c 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java
@@ -67,9 +67,16 @@ class DataLakeTest {
.build())
.build());
- // 3. Firehose Stream
+ // 3. Firehose Stream delivering under the Glue table location. The static
+ // prefix gets yyyy/MM/dd/HH/ appended like real Firehose, which stays
+ // inside the table location since Athena reads it recursively.
firehose.createDeliveryStream(software.amazon.awssdk.services.firehose.model.CreateDeliveryStreamRequest.builder()
.deliveryStreamName(STREAM_NAME)
+ .extendedS3DestinationConfiguration(software.amazon.awssdk.services.firehose.model.ExtendedS3DestinationConfiguration.builder()
+ .bucketARN("arn:aws:s3:::floci-firehose-results")
+ .roleARN("arn:aws:iam::000000000000:role/datalake-firehose-role")
+ .prefix(STREAM_NAME + "/")
+ .build())
.build());
}
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.java
index 5a57416528..6334012bde 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.java
@@ -198,6 +198,7 @@ class SesEventPublishingTest {
.s3DestinationConfiguration(S3DestinationConfiguration.builder()
.bucketARN("arn:aws:s3:::" + firehoseBucket)
.roleARN("arn:aws:iam::000000000000:role/sdk-evt-fh-role")
+ .prefix(firehoseStreamName + "/")
.build())
.build())
.deliveryStreamARN();
diff --git a/docs/services/firehose.md b/docs/services/firehose.md
index 6b188d53a6..5d96befbb5 100644
--- a/docs/services/firehose.md
+++ b/docs/services/firehose.md
@@ -26,7 +26,18 @@ Floci emulates Amazon Data Firehose for streaming data ingestion and delivery to
1. **Buffering**: Incoming records are buffered in memory.
2. **Automatic Flush**: Floci automatically flushes the buffer to S3 after every 5 records for immediate local feedback.
-3. **Format**: Records are flushed as raw NDJSON (newline-delimited JSON) to the `floci-firehose-results` bucket.
+3. **Format**: Records are flushed as raw NDJSON (newline-delimited JSON) to the bucket configured in the S3 destination (`floci-firehose-results` if the stream has no destination configuration).
+
+## S3 object keys
+
+Delivered objects are named `<evaluated prefix><streamName>-<versionId>-<yyyy-MM-dd-HH-mm-ss>-<uuid>` (no file extension), matching [AWS's object name format](https://docs.aws.amazon.com/firehose/latest/dev/s3-object-name.html):
+
+- Without a `Prefix`, the default prefix `yyyy/MM/dd/HH/` is used.
+- A `Prefix` without expressions gets `yyyy/MM/dd/HH/` appended by literal concatenation — no `/` is inserted, exactly like AWS (`legacy` → `legacy2026/07/13/14/...`).
+- [Custom prefix expressions](https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html) are evaluated: `!{timestamp:<pattern>}` (Java `DateTimeFormatter` pattern; all instances share the same instant) and `!{firehose:random-string}` (a fresh 11-character alphanumeric string per instance). When the prefix contains any expression, nothing is appended to it.
+- `CustomTimeZone` is honored for all timestamps; absent or invalid time zones fall back to UTC.
+
+Known deviations from AWS: timestamps are evaluated at flush time instead of the oldest buffered record's arrival time; expressions AWS would reject at create time (unknown namespaces, `!{partitionKeyFromQuery:...}`/`!{partitionKeyFromLambda:...}` dynamic partitioning keys, invalid patterns) are kept literally in the key; delivered content is never compressed, so no compression extension is ever added.
## Configuration
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
index c4f5702406..3421628f57 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
@@ -15,8 +15,8 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.nio.charset.StandardCharsets;
-import java.time.ZoneOffset;
-import java.time.ZonedDateTime;
+import java.time.Clock;
+import java.time.ZoneId;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
@@ -31,13 +31,16 @@ public class FirehoseService {
private final Map<String, List<byte[]>> buffers = new ConcurrentHashMap<>();
private final S3Service s3Service;
private final RegionResolver regionResolver;
+ private final Clock clock;
@Inject
- public FirehoseService(StorageFactory storageFactory, S3Service s3Service, RegionResolver regionResolver) {
+ public FirehoseService(StorageFactory storageFactory, S3Service s3Service, RegionResolver regionResolver,
+ Clock clock) {
this.streamStore = storageFactory.create("firehose", "streams.json",
new TypeReference<Map<String, DeliveryStreamDescription>>() {});
this.s3Service = s3Service;
this.regionResolver = regionResolver;
+ this.clock = clock;
}
public String createDeliveryStream(String name, S3Destination s3Config) {
@@ -142,6 +145,7 @@ public class FirehoseService {
if (update.getPrefix() != null) current.setPrefix(update.getPrefix());
if (update.getErrorOutputPrefix() != null) current.setErrorOutputPrefix(update.getErrorOutputPrefix());
if (update.getCompressionFormat() != null) current.setCompressionFormat(update.getCompressionFormat());
+ if (update.getCustomTimeZone() != null) current.setCustomTimeZone(update.getCustomTimeZone());
if (update.getBufferingHints() != null) current.setBufferingHints(update.getBufferingHints());
if (update.getEncryptionConfiguration() != null) current.setEncryptionConfiguration(update.getEncryptionConfiguration());
}
@@ -258,8 +262,10 @@ public class FirehoseService {
try {
String bucket = resolveBucket(stream);
- String prefix = resolvePrefix(stream);
- String key = prefix + UUID.randomUUID() + ".json";
+ S3Destination s3 = stream.s3Destination();
+ ZoneId zone = S3ObjectKeyResolver.resolveZone(s3 != null ? s3.getCustomTimeZone() : null);
+ String key = S3ObjectKeyResolver.resolveKey(s3 != null ? s3.getPrefix() : null,
+ stream.getDeliveryStreamName(), stream.getVersionId(), clock.instant(), zone);
ensureBucket(bucket);
@@ -288,21 +294,6 @@ public class FirehoseService {
return DEFAULT_BUCKET;
}
- private String resolvePrefix(DeliveryStreamDescription stream) {
- S3Destination s3 = stream.s3Destination();
- String prefix = (s3 != null && s3.getPrefix() != null) ? s3.getPrefix() : stream.getDeliveryStreamName() + "/";
-
- // Substitute time-based placeholders matching real Firehose
- ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
- prefix = prefix
- .replace("{year}", String.format("%04d", now.getYear()))
- .replace("{month}", String.format("%02d", now.getMonthValue()))
- .replace("{day}", String.format("%02d", now.getDayOfMonth()))
- .replace("{hour}", String.format("%02d", now.getHour()));
-
- return prefix.endsWith("/") ? prefix : prefix + "/";
- }
-
private void ensureBucket(String bucket) {
try {
s3Service.createBucket(bucket, regionResolver.getDefaultRegion());
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/S3ObjectKeyResolver.java b/src/main/java/io/github/hectorvent/floci/services/firehose/S3ObjectKeyResolver.java
new file mode 100644
index 0000000000..c4f6893240
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/S3ObjectKeyResolver.java
@@ -0,0 +1,114 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import org.jboss.logging.Logger;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Builds S3 object keys for Firehose deliveries the way AWS does (verified
+ * against real AWS, see https://github.com/floci-io/floci/issues/1857): the key
+ * is {@code <evaluated prefix><suffix>} where the prefix supports
+ * {@code !{timestamp:<DateTimeFormatter pattern>}} and
+ * {@code !{firehose:random-string}} expressions, {@code yyyy/MM/dd/HH/} is
+ * appended by literal concatenation when the prefix holds no expression at all
+ * (including the null/empty prefix), and the suffix is
+ * {@code <streamName>-<versionId>-<yyyy-MM-dd-HH-mm-ss>-<uuid>} with no file
+ * extension.
+ *
+ * Known deviations from AWS: timestamps are evaluated at delivery time instead
+ * of the oldest buffered record's arrival time, and expressions AWS would
+ * reject at create time (unknown namespaces, dynamic-partitioning keys,
+ * invalid patterns) are kept literally in the key instead of failing.
+ */
+final class S3ObjectKeyResolver {
+
+ private static final Logger LOG = Logger.getLogger(S3ObjectKeyResolver.class);
+ private static final Pattern EXPRESSION = Pattern.compile("!\\{(?<namespace>[^:}]*)(?::(?<argument>[^}]*))?}");
+ private static final DateTimeFormatter DEFAULT_TIME_PREFIX =
+ DateTimeFormatter.ofPattern("yyyy/MM/dd/HH/", Locale.ROOT);
+ private static final DateTimeFormatter SUFFIX_TIMESTAMP =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss", Locale.ROOT);
+ private static final String RANDOM_STRING_ALPHABET =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ private static final int RANDOM_STRING_LENGTH = 11;
+
+ private S3ObjectKeyResolver() {}
+
+ static String resolveKey(String prefix, String streamName, String versionId,
+ Instant deliveryTime, ZoneId zone) {
+ ZonedDateTime time = ZonedDateTime.ofInstant(deliveryTime, zone);
+ return evaluatePrefix(prefix, time) + objectNameSuffix(streamName, versionId, time);
+ }
+
+ static String evaluatePrefix(String prefix, ZonedDateTime time) {
+ StringBuilder out = new StringBuilder();
+ boolean expressionSeen = false;
+ Matcher matcher = EXPRESSION.matcher(prefix == null ? "" : prefix);
+ while (matcher.find()) {
+ expressionSeen = true;
+ matcher.appendReplacement(out, Matcher.quoteReplacement(evaluateExpression(matcher, time)));
+ }
+ matcher.appendTail(out);
+ if (!expressionSeen) {
+ out.append(DEFAULT_TIME_PREFIX.format(time));
+ }
+ return out.toString();
+ }
+
+ static String objectNameSuffix(String streamName, String versionId, ZonedDateTime time) {
+ return streamName + "-" + versionId + "-" + SUFFIX_TIMESTAMP.format(time) + "-" + UUID.randomUUID();
+ }
+
+ static ZoneId resolveZone(String customTimeZone) {
+ if (customTimeZone == null || customTimeZone.isBlank()) {
+ return ZoneOffset.UTC;
+ }
+ try {
+ return ZoneId.of(customTimeZone);
+ } catch (Exception e) {
+ LOG.warnv("Invalid CustomTimeZone {0}, evaluating S3 prefixes in UTC", customTimeZone);
+ return ZoneOffset.UTC;
+ }
+ }
+
+ private static String evaluateExpression(Matcher matcher, ZonedDateTime time) {
+ String namespace = matcher.group("namespace");
+ String argument = matcher.group("argument");
+ if ("timestamp".equals(namespace) && argument != null) {
+ return formatTimestamp(argument, matcher.group(), time);
+ }
+ if ("firehose".equals(namespace) && "random-string".equals(argument)) {
+ return randomString();
+ }
+ LOG.warnv("Unsupported Firehose prefix expression {0}, keeping it literally", matcher.group());
+ return matcher.group();
+ }
+
+ private static String formatTimestamp(String pattern, String expression, ZonedDateTime time) {
+ try {
+ return DateTimeFormatter.ofPattern(pattern, Locale.ROOT).format(time);
+ } catch (Exception e) {
+ LOG.warnv("Invalid pattern in Firehose prefix expression {0}, keeping it literally: {1}",
+ expression, e.getMessage());
+ return expression;
+ }
+ }
+
+ private static String randomString() {
+ StringBuilder random = new StringBuilder(RANDOM_STRING_LENGTH);
+ for (int i = 0; i < RANDOM_STRING_LENGTH; i++) {
+ random.append(RANDOM_STRING_ALPHABET.charAt(
+ ThreadLocalRandom.current().nextInt(RANDOM_STRING_ALPHABET.length())));
+ }
+ return random.toString();
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
index 17f926eb3b..77cd06e86d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
@@ -126,6 +126,8 @@ public class DeliveryStreamDescription {
private String errorOutputPrefix;
@JsonProperty("CompressionFormat")
private String compressionFormat;
+ @JsonProperty("CustomTimeZone")
+ private String customTimeZone;
@JsonProperty("BufferingHints")
private BufferingHints bufferingHints;
@JsonProperty("EncryptionConfiguration")
@@ -142,6 +144,8 @@ public class DeliveryStreamDescription {
public void setErrorOutputPrefix(String errorOutputPrefix) { this.errorOutputPrefix = errorOutputPrefix; }
public String getCompressionFormat() { return compressionFormat; }
public void setCompressionFormat(String compressionFormat) { this.compressionFormat = compressionFormat; }
+ public String getCustomTimeZone() { return customTimeZone; }
+ public void setCustomTimeZone(String customTimeZone) { this.customTimeZone = customTimeZone; }
public BufferingHints getBufferingHints() { return bufferingHints; }
public void setBufferingHints(BufferingHints bufferingHints) { this.bufferingHints = bufferingHints; }
public EncryptionConfiguration getEncryptionConfiguration() { return encryptionConfiguration; }
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java
index a8fa16ee89..406471215d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java
@@ -8,9 +8,13 @@ import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.jupiter.api.Assertions.assertTrue;
@QuarkusTest
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
@@ -320,4 +324,142 @@ class FirehoseExtendedS3IntegrationTest {
.statusCode(400)
.body("__type", equalTo("InvalidArgumentException"));
}
+
+ // Delivery-plane coverage: flushed objects follow the AWS key format
+ // <evaluated prefix><stream>-<version>-<yyyy-MM-dd-HH-mm-ss>-<uuid> observed
+ // on real AWS (https://github.com/floci-io/floci/issues/1857).
+
+ private static final String SUFFIX_REGEX =
+ "-1-\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
+ private static final String TIME_PREFIX_REGEX = "\\d{4}/\\d{2}/\\d{2}/\\d{2}/";
+
+ @Test
+ @Order(11)
+ void deliveryWithoutPrefixUsesAwsDefaultTimePrefix() {
+ String stream = "delivery-default-stream";
+ String bucket = "extended-s3-delivery-default";
+ createDeliveryStream(stream, bucket, "");
+ putFiveRecords(stream);
+
+ String key = firstDeliveredKey(bucket, "");
+ assertTrue(key.matches(TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);
+ }
+
+ @Test
+ @Order(12)
+ void deliveryAppendsTimePrefixToStaticPrefix() {
+ String stream = "delivery-static-stream";
+ String bucket = "extended-s3-delivery-static";
+ createDeliveryStream(stream, bucket, ", \"Prefix\": \"events/data/\"");
+ putFiveRecords(stream);
+
+ String key = firstDeliveredKey(bucket, "events/data/");
+ assertTrue(key.matches("events/data/" + TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);
+ }
+
+ @Test
+ @Order(13)
+ void deliveryConcatenatesTimePrefixWithoutInsertingSlash() {
+ String stream = "delivery-noslash-stream";
+ String bucket = "extended-s3-delivery-noslash";
+ createDeliveryStream(stream, bucket, ", \"Prefix\": \"legacy\"");
+ putFiveRecords(stream);
+
+ String key = firstDeliveredKey(bucket, "legacy");
+ assertTrue(key.matches("legacy" + TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);
+ }
+
+ @Test
+ @Order(14)
+ void deliveryEvaluatesTimestampExpressionWithoutAppendingDefault() {
+ String stream = "delivery-expr-stream";
+ String bucket = "extended-s3-delivery-expr";
+ createDeliveryStream(stream, bucket, ", \"Prefix\": \"data/!{timestamp:yyyy/MM/dd}/\"");
+ putFiveRecords(stream);
+
+ String key = firstDeliveredKey(bucket, "data/");
+ assertTrue(key.matches("data/\\d{4}/\\d{2}/\\d{2}/" + stream + SUFFIX_REGEX), key);
+ }
+
+ @Test
+ @Order(15)
+ void deliveryEvaluatesRandomStringExpressionWithoutAppendingDefault() {
+ String stream = "delivery-rand-stream";
+ String bucket = "extended-s3-delivery-rand";
+ createDeliveryStream(stream, bucket, ", \"Prefix\": \"rand/!{firehose:random-string}/\"");
+ putFiveRecords(stream);
+
+ String key = firstDeliveredKey(bucket, "rand/");
+ assertTrue(key.matches("rand/[A-Za-z0-9]{11}/" + stream + SUFFIX_REGEX), key);
+ }
+
+ @Test
+ @Order(16)
+ void customTimeZoneIsEchoedAndDeliveryKeepsAwsKeyShape() {
+ String stream = "delivery-tz-stream";
+ String bucket = "extended-s3-delivery-tz";
+ createDeliveryStream(stream, bucket,
+ ", \"Prefix\": \"events/\", \"CustomTimeZone\": \"Europe/Madrid\"");
+
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")
+ .body("{ \"DeliveryStreamName\": \"" + stream + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CustomTimeZone",
+ equalTo("Europe/Madrid"));
+
+ putFiveRecords(stream);
+ String key = firstDeliveredKey(bucket, "events/");
+ assertTrue(key.matches("events/" + TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);
+ }
+
+ private void createDeliveryStream(String streamName, String bucket, String extraDestinationJson) {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "ExtendedS3DestinationConfiguration": {
+ "RoleARN": "%s",
+ "BucketARN": "arn:aws:s3:::%s"%s
+ }
+ }
+ """.formatted(streamName, ROLE_ARN, bucket, extraDestinationJson))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ }
+
+ /** Five records reach DEFAULT_FLUSH_COUNT, so the batch triggers the automatic flush. */
+ private void putFiveRecords(String streamName) {
+ String data = Base64.getEncoder().encodeToString("{\"id\": 1}".getBytes(StandardCharsets.UTF_8));
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "PutRecordBatch")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "Records": [
+ {"Data": "%s"}, {"Data": "%s"}, {"Data": "%s"}, {"Data": "%s"}, {"Data": "%s"}
+ ]
+ }
+ """.formatted(streamName, data, data, data, data, data))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("FailedPutCount", equalTo(0));
+ }
+
+ private String firstDeliveredKey(String bucket, String listPrefix) {
+ return given().when().get("/" + bucket + "?prefix=" + listPrefix)
+ .then().statusCode(200)
+ .extract().xmlPath().getString("ListBucketResult.Contents[0].Key");
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
new file mode 100644
index 0000000000..605305e4ba
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
@@ -0,0 +1,116 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
+import io.github.hectorvent.floci.services.firehose.model.Record;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.testing.MutableClock;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyMap;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class FirehoseServiceTest {
+
+ private static final String UUID_REGEX = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
+
+ private FirehoseService firehoseService;
+ private S3Service s3Service;
+
+ @BeforeEach
+ void setUp() {
+ StorageFactory storageFactory = Mockito.mock(StorageFactory.class);
+ when(storageFactory.create(anyString(), anyString(), any()))
+ .thenReturn(new InMemoryStorage<>());
+ s3Service = Mockito.mock(S3Service.class);
+ firehoseService = new FirehoseService(storageFactory, s3Service,
+ new RegionResolver("us-east-1", "000000000000"), new MutableClock());
+ }
+
+ private void putRecordsUntilFlush(String streamName) {
+ for (int i = 0; i < 5; i++) {
+ Record record = new Record(("{\"n\":" + i + "}").getBytes(StandardCharsets.UTF_8));
+ firehoseService.putRecord(streamName, record);
+ }
+ }
+
+ private String deliveredKey(String expectedBucket) {
+ ArgumentCaptor<String> bucket = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);
+ verify(s3Service).putObject(bucket.capture(), key.capture(), any(byte[].class), anyString(), anyMap());
+ assertEquals(expectedBucket, bucket.getValue());
+ return key.getValue();
+ }
+
+ @Test
+ void deliversToDefaultBucketWithAwsShapedKey() {
+ firehoseService.createDeliveryStream("my-stream", null);
+ putRecordsUntilFlush("my-stream");
+
+ String key = deliveredKey("floci-firehose-results");
+ assertTrue(key.matches("2026/01/01/00/my-stream-1-2026-01-01-00-00-00-" + UUID_REGEX), key);
+ }
+
+ @Test
+ void staticPrefixGetsDefaultTimePrefixAppended() {
+ S3Destination s3 = new S3Destination();
+ s3.setBucketArn("arn:aws:s3:::custom-bucket");
+ s3.setPrefix("events/data/");
+ firehoseService.createDeliveryStream("my-stream", s3);
+ putRecordsUntilFlush("my-stream");
+
+ String key = deliveredKey("custom-bucket");
+ assertTrue(key.matches("events/data/2026/01/01/00/my-stream-1-2026-01-01-00-00-00-" + UUID_REGEX), key);
+ }
+
+ @Test
+ void customTimeZoneShiftsPrefixAndSuffix() {
+ S3Destination s3 = new S3Destination();
+ s3.setBucketArn("arn:aws:s3:::custom-bucket");
+ s3.setCustomTimeZone("Europe/Madrid");
+ firehoseService.createDeliveryStream("my-stream", s3);
+ putRecordsUntilFlush("my-stream");
+
+ String key = deliveredKey("custom-bucket");
+ assertTrue(key.matches("2026/01/01/01/my-stream-1-2026-01-01-01-00-00-" + UUID_REGEX), key);
+ }
+
+ @Test
+ void updateDestinationMergesCustomTimeZoneAndBumpsKeyVersion() {
+ S3Destination s3 = new S3Destination();
+ s3.setBucketArn("arn:aws:s3:::custom-bucket");
+ s3.setCustomTimeZone("Europe/Madrid");
+ firehoseService.createDeliveryStream("my-stream", s3);
+
+ S3Destination prefixOnly = new S3Destination();
+ prefixOnly.setPrefix("events/");
+ firehoseService.updateDestination("my-stream", "1", "destinationId-000000000001", prefixOnly);
+
+ DeliveryStreamDescription described = firehoseService.describeDeliveryStream("my-stream");
+ assertEquals("Europe/Madrid", described.s3Destination().getCustomTimeZone());
+ assertEquals("events/", described.s3Destination().getPrefix());
+
+ S3Destination timeZoneOnly = new S3Destination();
+ timeZoneOnly.setCustomTimeZone("Asia/Tokyo");
+ firehoseService.updateDestination("my-stream", "2", "destinationId-000000000001", timeZoneOnly);
+ assertEquals("Asia/Tokyo",
+ firehoseService.describeDeliveryStream("my-stream").s3Destination().getCustomTimeZone());
+
+ putRecordsUntilFlush("my-stream");
+ String key = deliveredKey("custom-bucket");
+ assertTrue(key.matches("events/2026/01/01/09/my-stream-3-2026-01-01-09-00-00-" + UUID_REGEX), key);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/S3ObjectKeyResolverTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/S3ObjectKeyResolverTest.java
new file mode 100644
index 0000000000..9598750f63
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/S3ObjectKeyResolverTest.java
@@ -0,0 +1,123 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class S3ObjectKeyResolverTest {
+
+ private static final Instant INSTANT = Instant.parse("2026-07-13T10:42:07Z");
+ private static final ZonedDateTime TIME = ZonedDateTime.ofInstant(INSTANT, ZoneOffset.UTC);
+ private static final String UUID_REGEX = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
+
+ @Test
+ void nullPrefixEvaluatesToDefaultTimePrefix() {
+ assertEquals("2026/07/13/10/", S3ObjectKeyResolver.evaluatePrefix(null, TIME));
+ }
+
+ @Test
+ void emptyPrefixEvaluatesToDefaultTimePrefix() {
+ assertEquals("2026/07/13/10/", S3ObjectKeyResolver.evaluatePrefix("", TIME));
+ }
+
+ @Test
+ void staticPrefixGetsDefaultTimePrefixAppended() {
+ assertEquals("events/data/2026/07/13/10/", S3ObjectKeyResolver.evaluatePrefix("events/data/", TIME));
+ }
+
+ @Test
+ void staticPrefixWithoutTrailingSlashIsConcatenatedLiterally() {
+ assertEquals("legacy2026/07/13/10/", S3ObjectKeyResolver.evaluatePrefix("legacy", TIME));
+ }
+
+ @Test
+ void timestampExpressionIsEvaluatedWithoutAppendingDefault() {
+ assertEquals("data/2026/07/13/", S3ObjectKeyResolver.evaluatePrefix("data/!{timestamp:yyyy/MM/dd}/", TIME));
+ }
+
+ @Test
+ void multipleTimestampExpressionsShareTheSameInstant() {
+ assertEquals("year=2026/hour=10/",
+ S3ObjectKeyResolver.evaluatePrefix("year=!{timestamp:yyyy}/hour=!{timestamp:HH}/", TIME));
+ }
+
+ @Test
+ void singleQuotedTextInTimestampPatternIsLiteral() {
+ assertEquals("year=2026/", S3ObjectKeyResolver.evaluatePrefix("!{timestamp:'year='yyyy}/", TIME));
+ }
+
+ @Test
+ void randomStringExpressionYieldsFreshElevenCharAlphanumerics() {
+ String evaluated = S3ObjectKeyResolver.evaluatePrefix(
+ "rand/!{firehose:random-string}/!{firehose:random-string}/", TIME);
+ assertTrue(evaluated.matches("rand/[A-Za-z0-9]{11}/[A-Za-z0-9]{11}/"), evaluated);
+ String[] parts = evaluated.split("/");
+ assertNotEquals(parts[1], parts[2]);
+ }
+
+ // Observed on real AWS but not documented: the docs' semantic rules say the
+ // default time prefix is appended whenever there is no *timestamp* expression,
+ // yet a prefix whose only expression is !{firehose:random-string} gets nothing
+ // appended (https://github.com/floci-io/floci/issues/1857).
+ @Test
+ void randomStringExpressionSuppressesDefaultTimePrefix() {
+ String evaluated = S3ObjectKeyResolver.evaluatePrefix("rand/!{firehose:random-string}/", TIME);
+ assertTrue(evaluated.matches("rand/[A-Za-z0-9]{11}/"), evaluated);
+ }
+
+ @Test
+ void dynamicPartitioningExpressionIsKeptLiterally() {
+ assertEquals("data/!{partitionKeyFromQuery:customerId}/",
+ S3ObjectKeyResolver.evaluatePrefix("data/!{partitionKeyFromQuery:customerId}/", TIME));
+ }
+
+ @Test
+ void unknownNamespaceIsKeptLiterally() {
+ assertEquals("data/!{bogus}/", S3ObjectKeyResolver.evaluatePrefix("data/!{bogus}/", TIME));
+ }
+
+ @Test
+ void invalidTimestampPatternIsKeptLiterally() {
+ assertEquals("data/!{timestamp:bbbb}/", S3ObjectKeyResolver.evaluatePrefix("data/!{timestamp:bbbb}/", TIME));
+ }
+
+ @Test
+ void unclosedExpressionIsLiteralAndStillGetsDefaultAppended() {
+ assertEquals("data/!{timestamp:yyyy2026/07/13/10/",
+ S3ObjectKeyResolver.evaluatePrefix("data/!{timestamp:yyyy", TIME));
+ }
+
+ @Test
+ void objectNameSuffixFollowsAwsFormat() {
+ String suffix = S3ObjectKeyResolver.objectNameSuffix("my-stream", "1", TIME);
+ assertTrue(suffix.matches("my-stream-1-2026-07-13-10-42-07-" + UUID_REGEX), suffix);
+ }
+
+ @Test
+ void resolveKeyConcatenatesEvaluatedPrefixAndSuffix() {
+ String key = S3ObjectKeyResolver.resolveKey("legacy", "my-stream", "3", INSTANT, ZoneOffset.UTC);
+ assertTrue(key.matches("legacy2026/07/13/10/my-stream-3-2026-07-13-10-42-07-" + UUID_REGEX), key);
+ }
+
+ @Test
+ void customTimeZoneShiftsPrefixAndSuffixAcrossTheDayBoundary() {
+ String key = S3ObjectKeyResolver.resolveKey(null, "s", "1",
+ Instant.parse("2026-01-01T23:30:00Z"), ZoneId.of("Europe/Madrid"));
+ assertTrue(key.matches("2026/01/02/00/s-1-2026-01-02-00-30-00-" + UUID_REGEX), key);
+ }
+
+ @Test
+ void resolveZoneFallsBackToUtc() {
+ assertEquals(ZoneOffset.UTC, S3ObjectKeyResolver.resolveZone(null));
+ assertEquals(ZoneOffset.UTC, S3ObjectKeyResolver.resolveZone(" "));
+ assertEquals(ZoneOffset.UTC, S3ObjectKeyResolver.resolveZone("Not/AZone"));
+ assertEquals(ZoneId.of("Europe/Madrid"), S3ObjectKeyResolver.resolveZone("Europe/Madrid"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java
index 96742db674..3e372eaaa7 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java
@@ -546,8 +546,8 @@ class SesEventPublishingV2IntegrationTest {
.header("X-Amz-Target", "Firehose_20150804.CreateDeliveryStream")
.body("""
{"DeliveryStreamName": "%s",
- "S3DestinationConfiguration": {"BucketARN": "arn:aws:s3:::%s"}}
- """.formatted(FIREHOSE_STREAM, FIREHOSE_BUCKET))
+ "S3DestinationConfiguration": {"BucketARN": "arn:aws:s3:::%s", "Prefix": "%s/"}}
+ """.formatted(FIREHOSE_STREAM, FIREHOSE_BUCKET, FIREHOSE_STREAM))
.when()
.post("/")
.then()
@@ -676,8 +676,8 @@ class SesEventPublishingV2IntegrationTest {
.header("X-Amz-Target", "Firehose_20150804.CreateDeliveryStream")
.body("""
{"DeliveryStreamName": "%s",
- "S3DestinationConfiguration": {"BucketARN": "arn:aws:s3:::%s"}}
- """.formatted(streamDisabled, FIREHOSE_BUCKET))
+ "S3DestinationConfiguration": {"BucketARN": "arn:aws:s3:::%s", "Prefix": "%s/"}}
+ """.formatted(streamDisabled, FIREHOSE_BUCKET, streamDisabled))
.when()
.post("/")
.then()