Light, fluffy, and always free - The AWS Local Emulator alternative
fix(firehose): build S3 object keys with AWS prefix expressions and name suffix (#1858)
Mihai Blidarua79e04e4ecd9
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.javaindex b53e754d8e..a249444f6c 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java@@ -67,9 +67,16 @@ class DataLakeTest {.build()).build());- // 3. Firehose Stream+ // 3. Firehose Stream delivering under the Glue table location. The static+ // prefix gets yyyy/MM/dd/HH/ appended like real Firehose, which stays+ // inside the table location since Athena reads it recursively.firehose.createDeliveryStream(software.amazon.awssdk.services.firehose.model.CreateDeliveryStreamRequest.builder().deliveryStreamName(STREAM_NAME)+ .extendedS3DestinationConfiguration(software.amazon.awssdk.services.firehose.model.ExtendedS3DestinationConfiguration.builder()+ .bucketARN("arn:aws:s3:::floci-firehose-results")+ .roleARN("arn:aws:iam::000000000000:role/datalake-firehose-role")+ .prefix(STREAM_NAME + "/")+ .build()).build());}diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.javaindex 5a57416528..6334012bde 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SesEventPublishingTest.java@@ -198,6 +198,7 @@ class SesEventPublishingTest {.s3DestinationConfiguration(S3DestinationConfiguration.builder().bucketARN("arn:aws:s3:::" + firehoseBucket).roleARN("arn:aws:iam::000000000000:role/sdk-evt-fh-role")+ .prefix(firehoseStreamName + "/").build()).build()).deliveryStreamARN();diff --git a/docs/services/firehose.md b/docs/services/firehose.mdindex 6b188d53a6..5d96befbb5 100644--- a/docs/services/firehose.md+++ b/docs/services/firehose.md@@ -26,7 +26,18 @@ Floci emulates Amazon Data Firehose for streaming data ingestion and delivery to1. **Buffering**: Incoming records are buffered in memory.2. **Automatic Flush**: Floci automatically flushes the buffer to S3 after every 5 records for immediate local feedback.-3. **Format**: Records are flushed as raw NDJSON (newline-delimited JSON) to the `floci-firehose-results` bucket.+3. **Format**: Records are flushed as raw NDJSON (newline-delimited JSON) to the bucket configured in the S3 destination (`floci-firehose-results` if the stream has no destination configuration).++## S3 object keys++Delivered objects are named `<evaluated prefix><streamName>-<versionId>-<yyyy-MM-dd-HH-mm-ss>-<uuid>` (no file extension), matching [AWS's object name format](https://docs.aws.amazon.com/firehose/latest/dev/s3-object-name.html):++- Without a `Prefix`, the default prefix `yyyy/MM/dd/HH/` is used.+- A `Prefix` without expressions gets `yyyy/MM/dd/HH/` appended by literal concatenation — no `/` is inserted, exactly like AWS (`legacy` → `legacy2026/07/13/14/...`).+- [Custom prefix expressions](https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html) are evaluated: `!{timestamp:<pattern>}` (Java `DateTimeFormatter` pattern; all instances share the same instant) and `!{firehose:random-string}` (a fresh 11-character alphanumeric string per instance). When the prefix contains any expression, nothing is appended to it.+- `CustomTimeZone` is honored for all timestamps; absent or invalid time zones fall back to UTC.++Known deviations from AWS: timestamps are evaluated at flush time instead of the oldest buffered record's arrival time; expressions AWS would reject at create time (unknown namespaces, `!{partitionKeyFromQuery:...}`/`!{partitionKeyFromLambda:...}` dynamic partitioning keys, invalid patterns) are kept literally in the key; delivered content is never compressed, so no compression extension is ever added.## Configurationdiff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.javaindex c4f5702406..3421628f57 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java@@ -15,8 +15,8 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.nio.charset.StandardCharsets;-import java.time.ZoneOffset;-import java.time.ZonedDateTime;+import java.time.Clock;+import java.time.ZoneId;import java.util.*;import java.util.concurrent.ConcurrentHashMap;@@ -31,13 +31,16 @@ public class FirehoseService {private final Map<String, List<byte[]>> buffers = new ConcurrentHashMap<>();private final S3Service s3Service;private final RegionResolver regionResolver;+ private final Clock clock;@Inject- public FirehoseService(StorageFactory storageFactory, S3Service s3Service, RegionResolver regionResolver) {+ public FirehoseService(StorageFactory storageFactory, S3Service s3Service, RegionResolver regionResolver,+ Clock clock) {this.streamStore = storageFactory.create("firehose", "streams.json",new TypeReference<Map<String, DeliveryStreamDescription>>() {});this.s3Service = s3Service;this.regionResolver = regionResolver;+ this.clock = clock;}public String createDeliveryStream(String name, S3Destination s3Config) {@@ -142,6 +145,7 @@ public class FirehoseService {if (update.getPrefix() != null) current.setPrefix(update.getPrefix());if (update.getErrorOutputPrefix() != null) current.setErrorOutputPrefix(update.getErrorOutputPrefix());if (update.getCompressionFormat() != null) current.setCompressionFormat(update.getCompressionFormat());+ if (update.getCustomTimeZone() != null) current.setCustomTimeZone(update.getCustomTimeZone());if (update.getBufferingHints() != null) current.setBufferingHints(update.getBufferingHints());if (update.getEncryptionConfiguration() != null) current.setEncryptionConfiguration(update.getEncryptionConfiguration());}@@ -258,8 +262,10 @@ public class FirehoseService {try {String bucket = resolveBucket(stream);- String prefix = resolvePrefix(stream);- String key = prefix + UUID.randomUUID() + ".json";+ S3Destination s3 = stream.s3Destination();+ ZoneId zone = S3ObjectKeyResolver.resolveZone(s3 != null ? s3.getCustomTimeZone() : null);+ String key = S3ObjectKeyResolver.resolveKey(s3 != null ? s3.getPrefix() : null,+ stream.getDeliveryStreamName(), stream.getVersionId(), clock.instant(), zone);ensureBucket(bucket);@@ -288,21 +294,6 @@ public class FirehoseService {return DEFAULT_BUCKET;}- private String resolvePrefix(DeliveryStreamDescription stream) {- S3Destination s3 = stream.s3Destination();- String prefix = (s3 != null && s3.getPrefix() != null) ? s3.getPrefix() : stream.getDeliveryStreamName() + "/";-- // Substitute time-based placeholders matching real Firehose- ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);- prefix = prefix- .replace("{year}", String.format("%04d", now.getYear()))- .replace("{month}", String.format("%02d", now.getMonthValue()))- .replace("{day}", String.format("%02d", now.getDayOfMonth()))- .replace("{hour}", String.format("%02d", now.getHour()));-- return prefix.endsWith("/") ? prefix : prefix + "/";- }-private void ensureBucket(String bucket) {try {s3Service.createBucket(bucket, regionResolver.getDefaultRegion());diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/S3ObjectKeyResolver.java b/src/main/java/io/github/hectorvent/floci/services/firehose/S3ObjectKeyResolver.javanew file mode 100644index 0000000000..c4f6893240--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/S3ObjectKeyResolver.java@@ -0,0 +1,114 @@+package io.github.hectorvent.floci.services.firehose;++import org.jboss.logging.Logger;++import java.time.Instant;+import java.time.ZoneId;+import java.time.ZoneOffset;+import java.time.ZonedDateTime;+import java.time.format.DateTimeFormatter;+import java.util.Locale;+import java.util.UUID;+import java.util.concurrent.ThreadLocalRandom;+import java.util.regex.Matcher;+import java.util.regex.Pattern;++/**+ * Builds S3 object keys for Firehose deliveries the way AWS does (verified+ * against real AWS, see https://github.com/floci-io/floci/issues/1857): the key+ * is {@code <evaluated prefix><suffix>} where the prefix supports+ * {@code !{timestamp:<DateTimeFormatter pattern>}} and+ * {@code !{firehose:random-string}} expressions, {@code yyyy/MM/dd/HH/} is+ * appended by literal concatenation when the prefix holds no expression at all+ * (including the null/empty prefix), and the suffix is+ * {@code <streamName>-<versionId>-<yyyy-MM-dd-HH-mm-ss>-<uuid>} with no file+ * extension.+ *+ * Known deviations from AWS: timestamps are evaluated at delivery time instead+ * of the oldest buffered record's arrival time, and expressions AWS would+ * reject at create time (unknown namespaces, dynamic-partitioning keys,+ * invalid patterns) are kept literally in the key instead of failing.+ */+final class S3ObjectKeyResolver {++ private static final Logger LOG = Logger.getLogger(S3ObjectKeyResolver.class);+ private static final Pattern EXPRESSION = Pattern.compile("!\\{(?<namespace>[^:}]*)(?::(?<argument>[^}]*))?}");+ private static final DateTimeFormatter DEFAULT_TIME_PREFIX =+ DateTimeFormatter.ofPattern("yyyy/MM/dd/HH/", Locale.ROOT);+ private static final DateTimeFormatter SUFFIX_TIMESTAMP =+ DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss", Locale.ROOT);+ private static final String RANDOM_STRING_ALPHABET =+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";+ private static final int RANDOM_STRING_LENGTH = 11;++ private S3ObjectKeyResolver() {}++ static String resolveKey(String prefix, String streamName, String versionId,+ Instant deliveryTime, ZoneId zone) {+ ZonedDateTime time = ZonedDateTime.ofInstant(deliveryTime, zone);+ return evaluatePrefix(prefix, time) + objectNameSuffix(streamName, versionId, time);+ }++ static String evaluatePrefix(String prefix, ZonedDateTime time) {+ StringBuilder out = new StringBuilder();+ boolean expressionSeen = false;+ Matcher matcher = EXPRESSION.matcher(prefix == null ? "" : prefix);+ while (matcher.find()) {+ expressionSeen = true;+ matcher.appendReplacement(out, Matcher.quoteReplacement(evaluateExpression(matcher, time)));+ }+ matcher.appendTail(out);+ if (!expressionSeen) {+ out.append(DEFAULT_TIME_PREFIX.format(time));+ }+ return out.toString();+ }++ static String objectNameSuffix(String streamName, String versionId, ZonedDateTime time) {+ return streamName + "-" + versionId + "-" + SUFFIX_TIMESTAMP.format(time) + "-" + UUID.randomUUID();+ }++ static ZoneId resolveZone(String customTimeZone) {+ if (customTimeZone == null || customTimeZone.isBlank()) {+ return ZoneOffset.UTC;+ }+ try {+ return ZoneId.of(customTimeZone);+ } catch (Exception e) {+ LOG.warnv("Invalid CustomTimeZone {0}, evaluating S3 prefixes in UTC", customTimeZone);+ return ZoneOffset.UTC;+ }+ }++ private static String evaluateExpression(Matcher matcher, ZonedDateTime time) {+ String namespace = matcher.group("namespace");+ String argument = matcher.group("argument");+ if ("timestamp".equals(namespace) && argument != null) {+ return formatTimestamp(argument, matcher.group(), time);+ }+ if ("firehose".equals(namespace) && "random-string".equals(argument)) {+ return randomString();+ }+ LOG.warnv("Unsupported Firehose prefix expression {0}, keeping it literally", matcher.group());+ return matcher.group();+ }++ private static String formatTimestamp(String pattern, String expression, ZonedDateTime time) {+ try {+ return DateTimeFormatter.ofPattern(pattern, Locale.ROOT).format(time);+ } catch (Exception e) {+ LOG.warnv("Invalid pattern in Firehose prefix expression {0}, keeping it literally: {1}",+ expression, e.getMessage());+ return expression;+ }+ }++ private static String randomString() {+ StringBuilder random = new StringBuilder(RANDOM_STRING_LENGTH);+ for (int i = 0; i < RANDOM_STRING_LENGTH; i++) {+ random.append(RANDOM_STRING_ALPHABET.charAt(+ ThreadLocalRandom.current().nextInt(RANDOM_STRING_ALPHABET.length())));+ }+ return random.toString();+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.javaindex 17f926eb3b..77cd06e86d 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java@@ -126,6 +126,8 @@ public class DeliveryStreamDescription {private String errorOutputPrefix;@JsonProperty("CompressionFormat")private String compressionFormat;+ @JsonProperty("CustomTimeZone")+ private String customTimeZone;@JsonProperty("BufferingHints")private BufferingHints bufferingHints;@JsonProperty("EncryptionConfiguration")@@ -142,6 +144,8 @@ public class DeliveryStreamDescription {public void setErrorOutputPrefix(String errorOutputPrefix) { this.errorOutputPrefix = errorOutputPrefix; }public String getCompressionFormat() { return compressionFormat; }public void setCompressionFormat(String compressionFormat) { this.compressionFormat = compressionFormat; }+ public String getCustomTimeZone() { return customTimeZone; }+ public void setCustomTimeZone(String customTimeZone) { this.customTimeZone = customTimeZone; }public BufferingHints getBufferingHints() { return bufferingHints; }public void setBufferingHints(BufferingHints bufferingHints) { this.bufferingHints = bufferingHints; }public EncryptionConfiguration getEncryptionConfiguration() { return encryptionConfiguration; }diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.javaindex a8fa16ee89..406471215d 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseExtendedS3IntegrationTest.java@@ -8,9 +8,13 @@ import org.junit.jupiter.api.Order;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.TestMethodOrder;+import java.nio.charset.StandardCharsets;+import java.util.Base64;+import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.equalTo;import static org.hamcrest.Matchers.notNullValue;+import static org.junit.jupiter.api.Assertions.assertTrue;@QuarkusTest@TestMethodOrder(MethodOrderer.OrderAnnotation.class)@@ -320,4 +324,142 @@ class FirehoseExtendedS3IntegrationTest {.statusCode(400).body("__type", equalTo("InvalidArgumentException"));}++ // Delivery-plane coverage: flushed objects follow the AWS key format+ // <evaluated prefix><stream>-<version>-<yyyy-MM-dd-HH-mm-ss>-<uuid> observed+ // on real AWS (https://github.com/floci-io/floci/issues/1857).++ private static final String SUFFIX_REGEX =+ "-1-\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";+ private static final String TIME_PREFIX_REGEX = "\\d{4}/\\d{2}/\\d{2}/\\d{2}/";++ @Test+ @Order(11)+ void deliveryWithoutPrefixUsesAwsDefaultTimePrefix() {+ String stream = "delivery-default-stream";+ String bucket = "extended-s3-delivery-default";+ createDeliveryStream(stream, bucket, "");+ putFiveRecords(stream);++ String key = firstDeliveredKey(bucket, "");+ assertTrue(key.matches(TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);+ }++ @Test+ @Order(12)+ void deliveryAppendsTimePrefixToStaticPrefix() {+ String stream = "delivery-static-stream";+ String bucket = "extended-s3-delivery-static";+ createDeliveryStream(stream, bucket, ", \"Prefix\": \"events/data/\"");+ putFiveRecords(stream);++ String key = firstDeliveredKey(bucket, "events/data/");+ assertTrue(key.matches("events/data/" + TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);+ }++ @Test+ @Order(13)+ void deliveryConcatenatesTimePrefixWithoutInsertingSlash() {+ String stream = "delivery-noslash-stream";+ String bucket = "extended-s3-delivery-noslash";+ createDeliveryStream(stream, bucket, ", \"Prefix\": \"legacy\"");+ putFiveRecords(stream);++ String key = firstDeliveredKey(bucket, "legacy");+ assertTrue(key.matches("legacy" + TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);+ }++ @Test+ @Order(14)+ void deliveryEvaluatesTimestampExpressionWithoutAppendingDefault() {+ String stream = "delivery-expr-stream";+ String bucket = "extended-s3-delivery-expr";+ createDeliveryStream(stream, bucket, ", \"Prefix\": \"data/!{timestamp:yyyy/MM/dd}/\"");+ putFiveRecords(stream);++ String key = firstDeliveredKey(bucket, "data/");+ assertTrue(key.matches("data/\\d{4}/\\d{2}/\\d{2}/" + stream + SUFFIX_REGEX), key);+ }++ @Test+ @Order(15)+ void deliveryEvaluatesRandomStringExpressionWithoutAppendingDefault() {+ String stream = "delivery-rand-stream";+ String bucket = "extended-s3-delivery-rand";+ createDeliveryStream(stream, bucket, ", \"Prefix\": \"rand/!{firehose:random-string}/\"");+ putFiveRecords(stream);++ String key = firstDeliveredKey(bucket, "rand/");+ assertTrue(key.matches("rand/[A-Za-z0-9]{11}/" + stream + SUFFIX_REGEX), key);+ }++ @Test+ @Order(16)+ void customTimeZoneIsEchoedAndDeliveryKeepsAwsKeyShape() {+ String stream = "delivery-tz-stream";+ String bucket = "extended-s3-delivery-tz";+ createDeliveryStream(stream, bucket,+ ", \"Prefix\": \"events/\", \"CustomTimeZone\": \"Europe/Madrid\"");++ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "DescribeDeliveryStream")+ .body("{ \"DeliveryStreamName\": \"" + stream + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DeliveryStreamDescription.Destinations[0].ExtendedS3DestinationDescription.CustomTimeZone",+ equalTo("Europe/Madrid"));++ putFiveRecords(stream);+ String key = firstDeliveredKey(bucket, "events/");+ assertTrue(key.matches("events/" + TIME_PREFIX_REGEX + stream + SUFFIX_REGEX), key);+ }++ private void createDeliveryStream(String streamName, String bucket, String extraDestinationJson) {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "ExtendedS3DestinationConfiguration": {+ "RoleARN": "%s",+ "BucketARN": "arn:aws:s3:::%s"%s+ }+ }+ """.formatted(streamName, ROLE_ARN, bucket, extraDestinationJson))+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }++ /** Five records reach DEFAULT_FLUSH_COUNT, so the batch triggers the automatic flush. */+ private void putFiveRecords(String streamName) {+ String data = Base64.getEncoder().encodeToString("{\"id\": 1}".getBytes(StandardCharsets.UTF_8));+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "PutRecordBatch")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "Records": [+ {"Data": "%s"}, {"Data": "%s"}, {"Data": "%s"}, {"Data": "%s"}, {"Data": "%s"}+ ]+ }+ """.formatted(streamName, data, data, data, data, data))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("FailedPutCount", equalTo(0));+ }++ private String firstDeliveredKey(String bucket, String listPrefix) {+ return given().when().get("/" + bucket + "?prefix=" + listPrefix)+ .then().statusCode(200)+ .extract().xmlPath().getString("ListBucketResult.Contents[0].Key");+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.javanew file mode 100644index 0000000000..605305e4ba--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java@@ -0,0 +1,116 @@+package io.github.hectorvent.floci.services.firehose;++import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;+import io.github.hectorvent.floci.services.firehose.model.Record;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.testing.MutableClock;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.mockito.ArgumentCaptor;+import org.mockito.Mockito;++import java.nio.charset.StandardCharsets;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyMap;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++class FirehoseServiceTest {++ private static final String UUID_REGEX = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";++ private FirehoseService firehoseService;+ private S3Service s3Service;++ @BeforeEach+ void setUp() {+ StorageFactory storageFactory = Mockito.mock(StorageFactory.class);+ when(storageFactory.create(anyString(), anyString(), any()))+ .thenReturn(new InMemoryStorage<>());+ s3Service = Mockito.mock(S3Service.class);+ firehoseService = new FirehoseService(storageFactory, s3Service,+ new RegionResolver("us-east-1", "000000000000"), new MutableClock());+ }++ private void putRecordsUntilFlush(String streamName) {+ for (int i = 0; i < 5; i++) {+ Record record = new Record(("{\"n\":" + i + "}").getBytes(StandardCharsets.UTF_8));+ firehoseService.putRecord(streamName, record);+ }+ }++ private String deliveredKey(String expectedBucket) {+ ArgumentCaptor<String> bucket = ArgumentCaptor.forClass(String.class);+ ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);+ verify(s3Service).putObject(bucket.capture(), key.capture(), any(byte[].class), anyString(), anyMap());+ assertEquals(expectedBucket, bucket.getValue());+ return key.getValue();+ }++ @Test+ void deliversToDefaultBucketWithAwsShapedKey() {+ firehoseService.createDeliveryStream("my-stream", null);+ putRecordsUntilFlush("my-stream");++ String key = deliveredKey("floci-firehose-results");+ assertTrue(key.matches("2026/01/01/00/my-stream-1-2026-01-01-00-00-00-" + UUID_REGEX), key);+ }++ @Test+ void staticPrefixGetsDefaultTimePrefixAppended() {+ S3Destination s3 = new S3Destination();+ s3.setBucketArn("arn:aws:s3:::custom-bucket");+ s3.setPrefix("events/data/");+ firehoseService.createDeliveryStream("my-stream", s3);+ putRecordsUntilFlush("my-stream");++ String key = deliveredKey("custom-bucket");+ assertTrue(key.matches("events/data/2026/01/01/00/my-stream-1-2026-01-01-00-00-00-" + UUID_REGEX), key);+ }++ @Test+ void customTimeZoneShiftsPrefixAndSuffix() {+ S3Destination s3 = new S3Destination();+ s3.setBucketArn("arn:aws:s3:::custom-bucket");+ s3.setCustomTimeZone("Europe/Madrid");+ firehoseService.createDeliveryStream("my-stream", s3);+ putRecordsUntilFlush("my-stream");++ String key = deliveredKey("custom-bucket");+ assertTrue(key.matches("2026/01/01/01/my-stream-1-2026-01-01-01-00-00-" + UUID_REGEX), key);+ }++ @Test+ void updateDestinationMergesCustomTimeZoneAndBumpsKeyVersion() {+ S3Destination s3 = new S3Destination();+ s3.setBucketArn("arn:aws:s3:::custom-bucket");+ s3.setCustomTimeZone("Europe/Madrid");+ firehoseService.createDeliveryStream("my-stream", s3);++ S3Destination prefixOnly = new S3Destination();+ prefixOnly.setPrefix("events/");+ firehoseService.updateDestination("my-stream", "1", "destinationId-000000000001", prefixOnly);++ DeliveryStreamDescription described = firehoseService.describeDeliveryStream("my-stream");+ assertEquals("Europe/Madrid", described.s3Destination().getCustomTimeZone());+ assertEquals("events/", described.s3Destination().getPrefix());++ S3Destination timeZoneOnly = new S3Destination();+ timeZoneOnly.setCustomTimeZone("Asia/Tokyo");+ firehoseService.updateDestination("my-stream", "2", "destinationId-000000000001", timeZoneOnly);+ assertEquals("Asia/Tokyo",+ firehoseService.describeDeliveryStream("my-stream").s3Destination().getCustomTimeZone());++ putRecordsUntilFlush("my-stream");+ String key = deliveredKey("custom-bucket");+ assertTrue(key.matches("events/2026/01/01/09/my-stream-3-2026-01-01-09-00-00-" + UUID_REGEX), key);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/S3ObjectKeyResolverTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/S3ObjectKeyResolverTest.javanew file mode 100644index 0000000000..9598750f63--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/S3ObjectKeyResolverTest.java@@ -0,0 +1,123 @@+package io.github.hectorvent.floci.services.firehose;++import org.junit.jupiter.api.Test;++import java.time.Instant;+import java.time.ZoneId;+import java.time.ZoneOffset;+import java.time.ZonedDateTime;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++class S3ObjectKeyResolverTest {++ private static final Instant INSTANT = Instant.parse("2026-07-13T10:42:07Z");+ private static final ZonedDateTime TIME = ZonedDateTime.ofInstant(INSTANT, ZoneOffset.UTC);+ private static final String UUID_REGEX = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";++ @Test+ void nullPrefixEvaluatesToDefaultTimePrefix() {+ assertEquals("2026/07/13/10/", S3ObjectKeyResolver.evaluatePrefix(null, TIME));+ }++ @Test+ void emptyPrefixEvaluatesToDefaultTimePrefix() {+ assertEquals("2026/07/13/10/", S3ObjectKeyResolver.evaluatePrefix("", TIME));+ }++ @Test+ void staticPrefixGetsDefaultTimePrefixAppended() {+ assertEquals("events/data/2026/07/13/10/", S3ObjectKeyResolver.evaluatePrefix("events/data/", TIME));+ }++ @Test+ void staticPrefixWithoutTrailingSlashIsConcatenatedLiterally() {+ assertEquals("legacy2026/07/13/10/", S3ObjectKeyResolver.evaluatePrefix("legacy", TIME));+ }++ @Test+ void timestampExpressionIsEvaluatedWithoutAppendingDefault() {+ assertEquals("data/2026/07/13/", S3ObjectKeyResolver.evaluatePrefix("data/!{timestamp:yyyy/MM/dd}/", TIME));+ }++ @Test+ void multipleTimestampExpressionsShareTheSameInstant() {+ assertEquals("year=2026/hour=10/",+ S3ObjectKeyResolver.evaluatePrefix("year=!{timestamp:yyyy}/hour=!{timestamp:HH}/", TIME));+ }++ @Test+ void singleQuotedTextInTimestampPatternIsLiteral() {+ assertEquals("year=2026/", S3ObjectKeyResolver.evaluatePrefix("!{timestamp:'year='yyyy}/", TIME));+ }++ @Test+ void randomStringExpressionYieldsFreshElevenCharAlphanumerics() {+ String evaluated = S3ObjectKeyResolver.evaluatePrefix(+ "rand/!{firehose:random-string}/!{firehose:random-string}/", TIME);+ assertTrue(evaluated.matches("rand/[A-Za-z0-9]{11}/[A-Za-z0-9]{11}/"), evaluated);+ String[] parts = evaluated.split("/");+ assertNotEquals(parts[1], parts[2]);+ }++ // Observed on real AWS but not documented: the docs' semantic rules say the+ // default time prefix is appended whenever there is no *timestamp* expression,+ // yet a prefix whose only expression is !{firehose:random-string} gets nothing+ // appended (https://github.com/floci-io/floci/issues/1857).+ @Test+ void randomStringExpressionSuppressesDefaultTimePrefix() {+ String evaluated = S3ObjectKeyResolver.evaluatePrefix("rand/!{firehose:random-string}/", TIME);+ assertTrue(evaluated.matches("rand/[A-Za-z0-9]{11}/"), evaluated);+ }++ @Test+ void dynamicPartitioningExpressionIsKeptLiterally() {+ assertEquals("data/!{partitionKeyFromQuery:customerId}/",+ S3ObjectKeyResolver.evaluatePrefix("data/!{partitionKeyFromQuery:customerId}/", TIME));+ }++ @Test+ void unknownNamespaceIsKeptLiterally() {+ assertEquals("data/!{bogus}/", S3ObjectKeyResolver.evaluatePrefix("data/!{bogus}/", TIME));+ }++ @Test+ void invalidTimestampPatternIsKeptLiterally() {+ assertEquals("data/!{timestamp:bbbb}/", S3ObjectKeyResolver.evaluatePrefix("data/!{timestamp:bbbb}/", TIME));+ }++ @Test+ void unclosedExpressionIsLiteralAndStillGetsDefaultAppended() {+ assertEquals("data/!{timestamp:yyyy2026/07/13/10/",+ S3ObjectKeyResolver.evaluatePrefix("data/!{timestamp:yyyy", TIME));+ }++ @Test+ void objectNameSuffixFollowsAwsFormat() {+ String suffix = S3ObjectKeyResolver.objectNameSuffix("my-stream", "1", TIME);+ assertTrue(suffix.matches("my-stream-1-2026-07-13-10-42-07-" + UUID_REGEX), suffix);+ }++ @Test+ void resolveKeyConcatenatesEvaluatedPrefixAndSuffix() {+ String key = S3ObjectKeyResolver.resolveKey("legacy", "my-stream", "3", INSTANT, ZoneOffset.UTC);+ assertTrue(key.matches("legacy2026/07/13/10/my-stream-3-2026-07-13-10-42-07-" + UUID_REGEX), key);+ }++ @Test+ void customTimeZoneShiftsPrefixAndSuffixAcrossTheDayBoundary() {+ String key = S3ObjectKeyResolver.resolveKey(null, "s", "1",+ Instant.parse("2026-01-01T23:30:00Z"), ZoneId.of("Europe/Madrid"));+ assertTrue(key.matches("2026/01/02/00/s-1-2026-01-02-00-30-00-" + UUID_REGEX), key);+ }++ @Test+ void resolveZoneFallsBackToUtc() {+ assertEquals(ZoneOffset.UTC, S3ObjectKeyResolver.resolveZone(null));+ assertEquals(ZoneOffset.UTC, S3ObjectKeyResolver.resolveZone(" "));+ assertEquals(ZoneOffset.UTC, S3ObjectKeyResolver.resolveZone("Not/AZone"));+ assertEquals(ZoneId.of("Europe/Madrid"), S3ObjectKeyResolver.resolveZone("Europe/Madrid"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.javaindex 96742db674..3e372eaaa7 100644--- a/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ses/SesEventPublishingV2IntegrationTest.java@@ -546,8 +546,8 @@ class SesEventPublishingV2IntegrationTest {.header("X-Amz-Target", "Firehose_20150804.CreateDeliveryStream").body("""{"DeliveryStreamName": "%s",- "S3DestinationConfiguration": {"BucketARN": "arn:aws:s3:::%s"}}- """.formatted(FIREHOSE_STREAM, FIREHOSE_BUCKET))+ "S3DestinationConfiguration": {"BucketARN": "arn:aws:s3:::%s", "Prefix": "%s/"}}+ """.formatted(FIREHOSE_STREAM, FIREHOSE_BUCKET, FIREHOSE_STREAM)).when().post("/").then()@@ -676,8 +676,8 @@ class SesEventPublishingV2IntegrationTest {.header("X-Amz-Target", "Firehose_20150804.CreateDeliveryStream").body("""{"DeliveryStreamName": "%s",- "S3DestinationConfiguration": {"BucketARN": "arn:aws:s3:::%s"}}- """.formatted(streamDisabled, FIREHOSE_BUCKET))+ "S3DestinationConfiguration": {"BucketARN": "arn:aws:s3:::%s", "Prefix": "%s/"}}+ """.formatted(streamDisabled, FIREHOSE_BUCKET, streamDisabled)).when().post("/").then()