Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(firehose): isolate streams by account and region (#3267)

Niels Kootstra141b15939f94
diff --git a/src/main/java/io/github/hectorvent/floci/services/eventbridge/EventBridgeInvoker.java b/src/main/java/io/github/hectorvent/floci/services/eventbridge/EventBridgeInvoker.java
index 460a4b9367..fbc25e9195 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eventbridge/EventBridgeInvoker.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eventbridge/EventBridgeInvoker.java
@@ -122,11 +122,17 @@ public class EventBridgeInvoker {
LOG.warnv("EventBridge Firehose target missing Firehose service: {0}", arn);
return;
}
- String streamName = arn.substring(
- arn.indexOf(":deliverystream/") + ":deliverystream/".length());
+ AwsArnUtils.Arn streamArn = AwsArnUtils.parse(arn);
+ String streamName = streamArn.resource().substring("deliverystream/".length());
// AWS puts the (input-transformed) event JSON as the record Data verbatim,
// without appending a newline; the delivery-side NDJSON flush handles separation.
- firehoseService.putRecord(streamName, new Record(payload.getBytes(StandardCharsets.UTF_8)));
+ Record record = new Record(payload.getBytes(StandardCharsets.UTF_8));
+ if (regionResolver == null || regionResolver.getRegion() == null) {
+ // Preserve the standalone/test mode where no request ownership context exists.
+ firehoseService.putRecord(streamName, record);
+ } else {
+ firehoseService.putRecord(streamArn.accountId(), streamArn.region(), streamName, record);
+ }
LOG.debugv("EventBridge delivered to Firehose: {0}", arn);
} else if (arn.contains(":events:") && arn.contains(":event-bus/")) {
if (eventBridgeService == null) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
index 3305881808..03a9062edd 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
@@ -5,7 +5,8 @@ import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
-import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.KinesisStreamSource;
@@ -61,7 +62,7 @@ public class FirehoseService implements ResourceProvider {
// busy source stream from monopolising the single flusher thread.
private static final int SOURCE_POLL_LIMIT = 500;
- private final StorageBackend<String, DeliveryStreamDescription> streamStore;
+ private final AccountAwareStorageBackend<DeliveryStreamDescription> streamStore;
private final Map<String, List<byte[]>> buffers = new ConcurrentHashMap<>();
private final Map<String, Instant> bufferSince = new ConcurrentHashMap<>();
private final S3Service s3Service;
@@ -82,7 +83,7 @@ public class FirehoseService implements ResourceProvider {
// This store holds only COMMITTED positions: a shard is advanced here once the
// records read up to that point have actually landed in S3. See
// pendingSourceIterators for why the two halves cannot be one map.
- private final StorageBackend<String, Map<String, String>> sourceIteratorStore;
+ private final AccountAwareStorageBackend<Map<String, String>> sourceIteratorStore;
// The advanced-but-not-yet-durable half of the checkpoint, deliberately in memory.
//
// Polled records do not go straight to S3; they go into buffers above and reach S3
@@ -105,6 +106,82 @@ public class FirehoseService implements ResourceProvider {
private final boolean flusherEnabled;
private final ScheduledExecutorService flushExecutor;
+ private String currentRegion() {
+ String region = regionResolver.getRegion();
+ return region == null || region.isBlank() ? regionResolver.getDefaultRegion() : region;
+ }
+
+ private String scopedKey(String streamName) {
+ return scopedKey(regionResolver.getAccountId(), currentRegion(), streamName);
+ }
+
+ private static String scopedKey(String accountId, String region, String streamName) {
+ return accountId + "/" + region + "/" + streamName;
+ }
+
+
+ private static String accountFromScopedKey(String key) {
+ return key.substring(0, key.indexOf('/'));
+ }
+
+ private static String logicalKeyFromScopedKey(String key) {
+ return key.substring(key.indexOf('/') + 1);
+ }
+
+ private static String regionFromScopedKey(String key) {
+ String logicalKey = logicalKeyFromScopedKey(key);
+ return logicalKey.substring(0, logicalKey.indexOf('/'));
+ }
+
+ private static String streamNameFromScopedKey(String key) {
+ String logicalKey = logicalKeyFromScopedKey(key);
+ return logicalKey.substring(logicalKey.indexOf('/') + 1);
+ }
+
+ private static boolean belongsTo(String accountId, String region, DeliveryStreamDescription stream) {
+ if (!accountId.equals(stream.getAccountId())) {
+ return false;
+ }
+ try {
+ AwsArnUtils.Arn arn = AwsArnUtils.parse(stream.getDeliveryStreamARN());
+ return accountId.equals(arn.accountId()) && region.equals(arn.region());
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
+ private Optional<DeliveryStreamDescription> streamGet(String key) {
+ String accountId = accountFromScopedKey(key);
+ String region = regionFromScopedKey(key);
+ String streamName = streamNameFromScopedKey(key);
+ return streamStore.getForAccountMigratingLegacyKeys(accountId, logicalKeyFromScopedKey(key),
+ List.of(streamName), stream -> belongsTo(accountId, region, stream), false);
+ }
+
+ private void streamPut(String key, DeliveryStreamDescription stream) {
+ streamStore.putForAccount(accountFromScopedKey(key), logicalKeyFromScopedKey(key), stream);
+ }
+
+ private void streamDelete(String key) {
+ streamStore.deleteForAccount(accountFromScopedKey(key), logicalKeyFromScopedKey(key));
+ }
+
+ private Optional<Map<String, String>> iteratorGet(String key) {
+ // Do not migrate the old account/name checkpoint key: its opaque iterator token has no
+ // region, so with same-named streams in multiple regions there is no safe way to tell
+ // which stream it belongs to. Leaving it untouched makes the regional stream start from
+ // its configured delivery timestamp instead of risking a cross-region checkpoint.
+ return sourceIteratorStore.getForAccount(accountFromScopedKey(key), logicalKeyFromScopedKey(key));
+ }
+
+ private void iteratorPut(String key, Map<String, String> value) {
+ sourceIteratorStore.putForAccount(accountFromScopedKey(key), logicalKeyFromScopedKey(key), value);
+ }
+
+ private void iteratorDelete(String key) {
+ sourceIteratorStore.deleteForAccount(accountFromScopedKey(key), logicalKeyFromScopedKey(key));
+ }
+
@Inject
public FirehoseService(StorageFactory storageFactory, S3Service s3Service, KinesisService kinesisService,
RegionResolver regionResolver, Clock clock, EmulatorConfig config) {
@@ -143,7 +220,7 @@ public class FirehoseService implements ResourceProvider {
// checkpoint out with everything else.
void onPreShutdown(@Observes ShutdownDelayInitiatedEvent ignored) {
flushExecutor.shutdownNow();
- buffers.keySet().forEach(this::flush);
+ buffers.keySet().forEach(this::flushByKey);
}
void tickSafely() {
@@ -162,7 +239,7 @@ public class FirehoseService implements ResourceProvider {
}
String streamName = entry.getKey();
try {
- DeliveryStreamDescription stream = describeDeliveryStream(streamName);
+ DeliveryStreamDescription stream = describeDeliveryStreamByKey(streamName);
Instant since = bufferSince.putIfAbsent(streamName, now);
if (since == null) {
since = now;
@@ -201,26 +278,34 @@ public class FirehoseService implements ResourceProvider {
public String createDeliveryStream(String name, S3Destination s3Config, List<DeliveryStreamDescription.Tag> tags,
String deliveryStreamType, KinesisStreamSource source) {
- return createDeliveryStream(regionResolver.getDefaultRegion(), name, s3Config, tags, deliveryStreamType,
- source);
+ return createDeliveryStream(regionResolver.getDefaultRegion(), regionResolver.getAccountId(), name, s3Config,
+ tags, deliveryStreamType, source);
}
public String createDeliveryStream(String region, String name, S3Destination s3Config,
List<DeliveryStreamDescription.Tag> tags, String deliveryStreamType,
KinesisStreamSource source) {
+ return createDeliveryStream(region, regionResolver.getAccountId(), name, s3Config, tags,
+ deliveryStreamType, source);
+ }
+
+ public String createDeliveryStream(String region, String accountId, String name, S3Destination s3Config,
+ List<DeliveryStreamDescription.Tag> tags, String deliveryStreamType,
+ KinesisStreamSource source) {
+ String streamKey = scopedKey(accountId, region, name);
if (name == null || name.isEmpty() || name.length() > 64 || !name.matches("[a-zA-Z0-9_.-]+")) {
throw new AwsException("InvalidArgumentException",
"Delivery stream name must be between 1 and 64 characters and contain only letters, numbers, underscores, hyphens, or periods.", 400);
}
- if (streamStore.get(name).isPresent()) {
+ if (streamGet(streamKey).isPresent()) {
throw new AwsException("ResourceInUseException",
"Delivery stream " + name + " already exists.", 409);
}
validateBufferingHints(s3Config);
DataFormatConversionValidator.validateEffective(s3Config);
- String arn = AwsArnUtils.Arn.of("firehose", region, regionResolver.getAccountId(),
+ String arn = AwsArnUtils.Arn.of("firehose", region, accountId,
"deliverystream/" + name).toString();
// CreateDeliveryStream's KinesisStreamSourceConfiguration carries only the ARN and
// role -- DeliveryStartTimestamp exists only on the Description shape, which AWS
@@ -232,20 +317,30 @@ public class FirehoseService implements ResourceProvider {
source.setDeliveryStartTimestamp(Instant.now());
}
DeliveryStreamDescription description = new DeliveryStreamDescription(name, arn, s3Config, source);
- description.setAccountId(regionResolver.getAccountId());
+ description.setAccountId(accountId);
description.setTags(tags);
if (deliveryStreamType != null && !deliveryStreamType.isBlank()) {
description.setDeliveryStreamType(deliveryStreamType);
}
- streamStore.put(name, description);
- buffers.put(name, Collections.synchronizedList(new ArrayList<>()));
+ streamPut(streamKey, description);
+ buffers.put(streamKey, Collections.synchronizedList(new ArrayList<>()));
LOG.infov("Created Firehose delivery stream: {0}", name);
warnIfConversionEnabled(name, s3Config);
return arn;
}
public void updateDestination(String name, String currentVersionId, String destinationId, S3Destination update) {
- DeliveryStreamDescription stream = describeDeliveryStream(name);
+ updateDestination(scopedKey(name), name, currentVersionId, destinationId, update);
+ }
+
+ public void updateDestination(String accountId, String region, String name, String currentVersionId,
+ String destinationId, S3Destination update) {
+ updateDestination(scopedKey(accountId, region, name), name, currentVersionId, destinationId, update);
+ }
+
+ private void updateDestination(String streamKey, String name, String currentVersionId, String destinationId, S3Destination update) {
+ DeliveryStreamDescription stream = streamGet(streamKey)
+ .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Delivery stream not found: " + name, 400));
if (!stream.getVersionId().equals(currentVersionId)) {
throw new AwsException("ConcurrentModificationException",
"Cannot update firehose: " + name + " since the current version id: " + stream.getVersionId()
@@ -279,7 +374,7 @@ public class FirehoseService implements ResourceProvider {
}
stream.setVersionId(String.valueOf(parseVersionId(stream.getVersionId()) + 1));
stream.setLastUpdateTimestamp(java.time.Instant.now());
- streamStore.put(name, stream);
+ streamPut(streamKey, stream);
LOG.infov("Updated destination {0} of Firehose delivery stream {1}", destinationId, name);
warnIfConversionEnabled(name, stream.s3Destination());
}
@@ -324,7 +419,7 @@ public class FirehoseService implements ResourceProvider {
effectiveKeyType,
effectiveKeyType.equals("CUSTOMER_MANAGED_CMK") ? keyArn : null,
"ENABLED"));
- streamStore.put(name, stream);
+ streamPut(scopedKey(name), stream);
}
public void stopDeliveryStreamEncryption(String name) {
@@ -339,7 +434,7 @@ public class FirehoseService implements ResourceProvider {
stream.setDeliveryStreamEncryptionConfiguration(
new DeliveryStreamDescription.DeliveryStreamEncryptionConfiguration(
keyType, keyArn, "DISABLED"));
- streamStore.put(name, stream);
+ streamPut(scopedKey(name), stream);
}
// A corrupt persisted version can only reach here when the caller echoed it
@@ -470,7 +565,7 @@ public class FirehoseService implements ResourceProvider {
List<DeliveryStreamDescription.Tag> newTags = new ArrayList<>();
tagMap.forEach((k, v) -> newTags.add(new DeliveryStreamDescription.Tag(k, v)));
stream.setTags(newTags);
- streamStore.put(name, stream);
+ streamPut(scopedKey(name), stream);
LOG.infov("Tagged Firehose delivery stream {0}: {1}", name, tagsToTag);
}
@@ -483,7 +578,7 @@ public class FirehoseService implements ResourceProvider {
}
}
stream.setTags(newTags);
- streamStore.put(name, stream);
+ streamPut(scopedKey(name), stream);
LOG.infov("Untagged Firehose delivery stream {0}: {1}", name, tagKeys);
}
@@ -508,9 +603,25 @@ public class FirehoseService implements ResourceProvider {
}
public DeliveryStreamDescription describeDeliveryStream(String name) {
- DeliveryStreamDescription stream = streamStore.get(name)
+ return describeDeliveryStream(scopedKey(name), name);
+ }
+
+ public DeliveryStreamDescription describeDeliveryStream(String accountId, String region, String name) {
+ return describeDeliveryStream(scopedKey(accountId, region, name), name);
+ }
+
+ private DeliveryStreamDescription describeDeliveryStream(String streamKey, String name) {
+ return describeDeliveryStreamByKey(streamKey, name);
+ }
+
+ private DeliveryStreamDescription describeDeliveryStreamByKey(String streamKey) {
+ return describeDeliveryStreamByKey(streamKey, streamKey.substring(streamKey.lastIndexOf('/') + 1));
+ }
+
+ private DeliveryStreamDescription describeDeliveryStreamByKey(String streamKey, String streamName) {
+ DeliveryStreamDescription stream = streamGet(streamKey)
.orElseThrow(() -> new AwsException("ResourceNotFoundException",
- "Delivery stream not found: " + name, 400));
+ "Delivery stream not found: " + streamName, 400));
// Normalizes streams persisted before required output members existed.
if (stream.s3Destination() != null) {
stream.s3Destination().applyDefaults();
@@ -519,20 +630,26 @@ public class FirehoseService implements ResourceProvider {
}
public void deleteDeliveryStream(String name) {
- describeDeliveryStream(name);
- streamStore.delete(name);
+ deleteDeliveryStream(regionResolver.getAccountId(), currentRegion(), name);
+ }
+
+ public void deleteDeliveryStream(String accountId, String region, String name) {
+ String streamKey = scopedKey(accountId, region, name);
+ describeDeliveryStream(streamKey, name);
+ streamDelete(streamKey);
// Pending records are discarded, not flushed: verified against real AWS
// (2026-07-13, eu-west-1) — 3 records buffered under a 300s/5MB hint never
// reached the bucket after DeleteDeliveryStream completed.
- buffers.remove(name);
- bufferSince.remove(name);
- pendingSourceIterators.remove(name);
- sourceIteratorStore.delete(name);
+ buffers.remove(streamKey);
+ bufferSince.remove(streamKey);
+ pendingSourceIterators.remove(streamKey);
+ iteratorDelete(streamKey);
LOG.infov("Deleted Firehose delivery stream: {0}", name);
}
public List<String> listDeliveryStreams() {
- return streamStore.scan(k -> true).stream()
+ String regionPrefix = currentRegion() + "/";
+ return streamStore.scan(k -> k.startsWith(regionPrefix)).stream()
.map(DeliveryStreamDescription::getDeliveryStreamName).toList();
}
@@ -554,14 +671,18 @@ public class FirehoseService implements ResourceProvider {
* records does not backfill them.
*/
void pollKinesisSources() {
- for (DeliveryStreamDescription stream : streamStore.scan(k -> true)) {
+ for (AccountAwareStorageBackend.AccountEntry<DeliveryStreamDescription> entry
+ : streamStore.scanAllAccountEntries(k -> true)) {
+ DeliveryStreamDescription stream = entry.value();
KinesisStreamSource source = stream.getSource() == null
? null : stream.getSource().getKinesisStreamSourceDescription();
if (source == null || source.getKinesisStreamArn() == null) {
continue;
}
try {
- pollKinesisSource(stream.getDeliveryStreamName(), source);
+ AwsArnUtils.Arn ownerArn = AwsArnUtils.parse(stream.getDeliveryStreamARN());
+ pollKinesisSource(scopedKey(ownerArn.accountId(), ownerArn.region(), stream.getDeliveryStreamName()),
+ ownerArn.accountId(), stream.getDeliveryStreamName(), source);
} catch (Exception e) {
// A source stream that was deleted (or was never created) is the caller's
// business, not a reason to stop polling every other delivery stream.
@@ -571,29 +692,33 @@ public class FirehoseService implements ResourceProvider {
}
}
- private void pollKinesisSource(String deliveryStreamName, KinesisStreamSource source) {
+ private void pollKinesisSource(String streamKey, String accountId, String deliveryStreamName,
+ KinesisStreamSource source) {
AwsArnUtils.Arn arn = AwsArnUtils.parse(source.getKinesisStreamArn());
String sourceName = arn.resource().startsWith("stream/")
? arn.resource().substring("stream/".length())
: arn.resource();
String region = arn.region() == null || arn.region().isBlank()
? regionResolver.getDefaultRegion() : arn.region();
- KinesisStream sourceStream = kinesisService.describeStream(sourceName, region);
+ String sourceAccountId = arn.accountId() == null || arn.accountId().isBlank()
+ ? accountId : arn.accountId();
+ KinesisStream sourceStream = kinesisService.describeStreamForAccount(sourceAccountId, sourceName, region);
for (KinesisShard shard : sourceStream.getShards()) {
// Where to read from: the pending position if a flush still owes these
// records to S3, otherwise the committed one. Reading pending is what stops
// a second poll re-reading what the first already buffered; committing only
// on flush is what stops the durable checkpoint outrunning delivery.
- String iterator = sourceIteratorsInUse(deliveryStreamName).get(shard.getShardId());
+ String iterator = sourceIteratorsInUse(streamKey).get(shard.getShardId());
if (iterator == null) {
Instant start = source.getDeliveryStartTimestamp();
iterator = start == null
- ? kinesisService.getShardIterator(sourceName, shard.getShardId(),
+ ? kinesisService.getShardIteratorForAccount(sourceAccountId, sourceName, shard.getShardId(),
"TRIM_HORIZON", null, region)
- : kinesisService.getShardIterator(sourceName, shard.getShardId(),
+ : kinesisService.getShardIteratorForAccount(sourceAccountId, sourceName, shard.getShardId(),
"AT_TIMESTAMP", null, start.toEpochMilli(), region);
}
- Map<String, Object> page = kinesisService.getRecords(iterator, SOURCE_POLL_LIMIT, region);
+ Map<String, Object> page = kinesisService.getRecordsForAccount(sourceAccountId, iterator,
+ SOURCE_POLL_LIMIT, region);
@SuppressWarnings("unchecked")
List<KinesisRecord> records = (List<KinesisRecord>) page.get("Records");
String next = (String) page.get("NextShardIterator");
@@ -602,66 +727,74 @@ public class FirehoseService implements ResourceProvider {
if (records == null || records.isEmpty()) {
// Nothing was buffered, so this advance owes S3 nothing and is safe to
// park as pending on its own.
- advanceSourceIterator(deliveryStreamName, shardId, advanced);
+ advanceSourceIterator(streamKey, shardId, advanced);
continue;
}
// Through putRecordBatch so source records buffer, and trigger a flush, on
// exactly the same terms as records handed to PutRecord directly. The
// advance runs under the buffer lock together with the records it covers, so
// no flush can ever see one without the other.
- putRecordBatch(deliveryStreamName, records.stream()
+ putRecordBatch(streamKey, deliveryStreamName, records.stream()
.map(record -> new Record(record.getData()))
.toList(),
- () -> advanceSourceIterator(deliveryStreamName, shardId, advanced));
+ () -> advanceSourceIterator(streamKey, shardId, advanced));
}
}
/** The position the poller reads from: pending if a flush still owes it, else committed. */
- private Map<String, String> sourceIteratorsInUse(String deliveryStreamName) {
- Map<String, String> pending = pendingSourceIterators.get(deliveryStreamName);
- return pending != null ? pending : sourceIteratorStore.get(deliveryStreamName).orElseGet(Map::of);
+ private Map<String, String> sourceIteratorsInUse(String streamKey) {
+ Map<String, String> pending = pendingSourceIterators.get(streamKey);
+ return pending != null ? pending : iteratorGet(streamKey).orElseGet(Map::of);
}
// Rebased on whatever is current rather than on a map carried across the shard loop,
// so that a flush which discarded pending (because its S3 write failed) rolls the
// other shards back to committed instead of having this poll re-assert them.
- private void advanceSourceIterator(String deliveryStreamName, String shardId, String iterator) {
- Map<String, String> advanced = new LinkedHashMap<>(sourceIteratorsInUse(deliveryStreamName));
+ private void advanceSourceIterator(String streamKey, String shardId, String iterator) {
+ Map<String, String> advanced = new LinkedHashMap<>(sourceIteratorsInUse(streamKey));
advanced.put(shardId, iterator);
- pendingSourceIterators.put(deliveryStreamName, Collections.unmodifiableMap(advanced));
+ pendingSourceIterators.put(streamKey, Collections.unmodifiableMap(advanced));
}
// Merged rather than written whole: two flushes can complete out of order, and a
// whole-map write would then drop a shard's newer committed position. Merging can
// still move one shard backwards, which costs a duplicate and never a skip.
- private void commitSourceIterators(String deliveryStreamName, Map<String, String> checkpoint) {
+ private void commitSourceIterators(String streamKey, Map<String, String> checkpoint) {
if (checkpoint == null || checkpoint.isEmpty()) {
return;
}
// Copied out and written back whole: the store may hand back its own instance,
// and a persistent backend only learns of a change through put().
Map<String, String> committed =
- new LinkedHashMap<>(sourceIteratorStore.get(deliveryStreamName).orElseGet(Map::of));
+ new LinkedHashMap<>(iteratorGet(streamKey).orElseGet(Map::of));
committed.putAll(checkpoint);
- sourceIteratorStore.put(deliveryStreamName, committed);
+ iteratorPut(streamKey, committed);
}
public void putRecord(String streamName, Record record) {
- putRecordBatch(streamName, List.of(record));
+ putRecordBatch(scopedKey(streamName), streamName, List.of(record), null);
+ }
+
+ public void putRecord(String accountId, String region, String streamName, Record record) {
+ putRecordBatch(scopedKey(accountId, region, streamName), streamName, List.of(record), null);
}
public void putRecordBatch(String streamName, List<Record> records) {
- putRecordBatch(streamName, records, null);
+ putRecordBatch(scopedKey(streamName), streamName, records, null);
+ }
+
+ public void putRecordBatch(String accountId, String region, String streamName, List<Record> records) {
+ putRecordBatch(scopedKey(accountId, region, streamName), streamName, records, null);
}
/**
* @param checkpointAdvance run under the buffer lock once the records are buffered,
* or null for records that are not read from a source stream.
*/
- private void putRecordBatch(String streamName, List<Record> records, Runnable checkpointAdvance) {
- DeliveryStreamDescription stream = describeDeliveryStream(streamName);
+ private void putRecordBatch(String streamKey, String streamName, List<Record> records, Runnable checkpointAdvance) {
+ DeliveryStreamDescription stream = describeDeliveryStreamByKey(streamKey);
List<byte[]> buffer = buffers.computeIfAbsent(
- streamName, k -> Collections.synchronizedList(new ArrayList<>()));
+ streamKey, k -> Collections.synchronizedList(new ArrayList<>()));
long bufferedBytes = 0;
int bufferedCount;
// Records, their buffering-start timestamp and any source checkpoint advance move
@@ -673,7 +806,7 @@ public class FirehoseService implements ResourceProvider {
if (checkpointAdvance != null) {
checkpointAdvance.run();
}
- bufferSince.putIfAbsent(streamName, clock.instant());
+ bufferSince.putIfAbsent(streamKey, clock.instant());
bufferedCount = buffer.size();
for (byte[] data : buffer) {
bufferedBytes += data.length;
@@ -681,7 +814,7 @@ public class FirehoseService implements ResourceProvider {
}
if ((flushRecordCount > 0 && bufferedCount >= flushRecordCount)
|| bufferedBytes >= bufferingSizeLimitBytes(stream)) {
- flush(streamName, stream);
+ flush(streamKey, stream);
}
}
@@ -694,7 +827,15 @@ public class FirehoseService implements ResourceProvider {
}
public void flush(String streamName) {
- streamStore.get(streamName).ifPresent(stream -> flush(streamName, stream));
+ flushByKey(scopedKey(streamName));
+ }
+
+ public void flush(String accountId, String region, String streamName) {
+ flushByKey(scopedKey(accountId, region, streamName));
+ }
+
+ private void flushByKey(String streamKey) {
+ streamGet(streamKey).ifPresent(stream -> flush(streamKey, stream));
}
private void flush(String streamName, DeliveryStreamDescription stream) {
@@ -779,7 +920,7 @@ public class FirehoseService implements ResourceProvider {
@Override
public List<ExplorerResource> getResources() {
List<ExplorerResource> resources = new ArrayList<>();
- for (DeliveryStreamDescription stream : streamStore.scan(k -> true)) {
+ for (DeliveryStreamDescription stream : streamStore.scanAllAccounts()) {
String arn = stream.getDeliveryStreamARN();
if (arn == null) {
continue;
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
index 75bde14c4e..bfab7057c6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
@@ -179,6 +179,10 @@ public class KinesisService implements ResourceProvider {
return resolveStream(streamName, region);
}
+ public KinesisStream describeStreamForAccount(String accountId, String streamName, String region) {
+ return resolveStreamForAccount(accountId, streamName, region);
+ }
+
public KinesisConsumer registerStreamConsumer(String streamArn, String consumerName, String region) {
String consumerArn = streamArn + "/consumer/" + consumerName + ":" + System.currentTimeMillis();
KinesisConsumer consumer = new KinesisConsumer(consumerName, consumerArn, streamArn);
@@ -860,11 +864,19 @@ public class KinesisService implements ResourceProvider {
public String getShardIteratorForAccount(String accountId, String streamName, String shardId,
String type, String sequenceNumber, String region) {
+ return getShardIteratorForAccount(accountId, streamName, shardId, type, sequenceNumber,
+ null, region);
+ }
+
+ public String getShardIteratorForAccount(String accountId, String streamName, String shardId,
+ String type, String sequenceNumber, Long timestampMillis,
+ String region) {
KinesisStream stream = resolveStreamForAccount(accountId, streamName, region);
- String raw = String.format("%s|%s|%s|%s|%d|",
+ String raw = String.format("%s|%s|%s|%s|%d|%s",
streamName, shardId, type,
sequenceNumber != null ? sequenceNumber : "",
- iteratorStartIndex(stream, shardId, type));
+ iteratorStartIndex(stream, shardId, type),
+ timestampMillis != null ? timestampMillis.toString() : "");
return Base64.getEncoder().encodeToString(raw.getBytes(StandardCharsets.UTF_8));
}
@@ -876,6 +888,14 @@ public class KinesisService implements ResourceProvider {
String type = parts[2];
String startSeq = parts[3];
int lastIndex = parseIteratorIndex(parts[4]);
+ Long timestampMillis = null;
+ if (parts.length >= 6 && !parts[5].isEmpty()) {
+ try {
+ timestampMillis = Long.parseLong(parts[5]);
+ } catch (NumberFormatException e) {
+ throw new AwsException("InvalidArgumentException", "Invalid timestamp in shard iterator", 400);
+ }
+ }
KinesisStream stream = resolveStreamForAccount(accountId, streamName, region);
KinesisShard shard = stream.getShards().stream()
@@ -895,6 +915,19 @@ public class KinesisService implements ResourceProvider {
break;
}
}
+ } else if ("AT_TIMESTAMP".equals(type)) {
+ if (timestampMillis == null) {
+ throw new AwsException("InvalidArgumentException",
+ "AT_TIMESTAMP iterator requires a Timestamp", 400);
+ }
+ startIndex = allRecords.size();
+ for (int i = 0; i < allRecords.size(); i++) {
+ Instant arrival = allRecords.get(i).getApproximateArrivalTimestamp();
+ if (arrival != null && arrival.toEpochMilli() >= timestampMillis) {
+ startIndex = i;
+ break;
+ }
+ }
}
int max = limit != null ? Math.min(limit, 1000) : 1000;
diff --git a/src/main/java/io/github/hectorvent/floci/services/ses/SesEventPublisher.java b/src/main/java/io/github/hectorvent/floci/services/ses/SesEventPublisher.java
index e2acb57ce9..e90909444c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ses/SesEventPublisher.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ses/SesEventPublisher.java
@@ -192,7 +192,8 @@ public class SesEventPublisher {
try {
Record record = new Record();
record.setData(payloadJson.getBytes(StandardCharsets.UTF_8));
- firehoseService.putRecord(streamName, record);
+ AwsArnUtils.Arn parsedArn = AwsArnUtils.parse(streamArn);
+ firehoseService.putRecord(parsedArn.accountId(), parsedArn.region(), streamName, record);
} catch (Exception e) {
LOG.warnf(e, "SES event publish to Firehose stream %s skipped", streamName);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
index 0fa591423f..525f54fa4c 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.firehose;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
@@ -23,18 +24,23 @@ import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyMap;
import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -48,6 +54,7 @@ class FirehoseServiceTest {
private FirehoseService firehoseService;
private KinesisService kinesisService;
private StorageFactory storageFactory;
+ private Map<String, AccountAwareStorageBackend<?>> backends;
private S3Service s3Service;
private MutableClock clock;
@@ -59,7 +66,7 @@ class FirehoseServiceTest {
// the map FirehoseService.scan() reads as delivery streams; a fresh instance per
// create() call would make a second service over "the same storage" impossible
// to build, which is what the restart test needs.
- Map<String, AccountAwareStorageBackend<?>> backends = new HashMap<>();
+ backends = new HashMap<>();
when(storageFactory.create(anyString(), anyString(), any()))
.thenAnswer(invocation -> backends.computeIfAbsent(
invocation.getArgument(0) + "/" + invocation.getArgument(1),
@@ -138,6 +145,85 @@ class FirehoseServiceTest {
anyMap(), any(PutObjectOptions.class));
}
+ @Test
+ void migratesLegacyAccountAndNameKeysOnlyForMatchingOwner() {
+ DeliveryStreamDescription legacy = new DeliveryStreamDescription(
+ "legacy", "arn:aws:firehose:us-east-1:111111111111:deliverystream/legacy", null, null);
+ legacy.setAccountId("111111111111");
+ AccountAwareStorageBackend<DeliveryStreamDescription> streams =
+ (AccountAwareStorageBackend<DeliveryStreamDescription>) backends.get("firehose/streams.json");
+ streams.putForAccount("111111111111", "legacy", legacy);
+
+ assertEquals("legacy", firehoseService.describeDeliveryStream("111111111111", "us-east-1", "legacy")
+ .getDeliveryStreamName());
+ assertFalse(streams.getForAccount("111111111111", "legacy").isPresent());
+ assertTrue(streams.getForAccount("111111111111", "us-east-1/legacy").isPresent());
+
+ DeliveryStreamDescription wrongRegion = new DeliveryStreamDescription(
+ "wrong-region", "arn:aws:firehose:eu-west-1:111111111111:deliverystream/wrong-region", null, null);
+ wrongRegion.setAccountId("111111111111");
+ streams.putForAccount("111111111111", "wrong-region", wrongRegion);
+ assertThrows(AwsException.class,
+ () -> firehoseService.describeDeliveryStream("111111111111", "us-east-1", "wrong-region"),
+ "legacy entries from another region must not be adopted");
+ }
+
+ @Test
+ void isolatesSameStreamNameAcrossAccountsAndRegionsIncludingFlushAndDelete() {
+ String firstArn = firehoseService.createDeliveryStream("us-east-1", "111111111111", "shared", null,
+ List.of(), null, null);
+ String secondArn = firehoseService.createDeliveryStream("eu-west-1", "222222222222", "shared", null,
+ List.of(), null, null);
+
+ assertTrue(firstArn.contains(":us-east-1:111111111111:deliverystream/shared"));
+ assertTrue(secondArn.contains(":eu-west-1:222222222222:deliverystream/shared"));
+
+ firehoseService.putRecord("111111111111", "us-east-1", "shared",
+ new Record("first".getBytes(StandardCharsets.UTF_8)));
+ firehoseService.putRecord("222222222222", "eu-west-1", "shared",
+ new Record("second".getBytes(StandardCharsets.UTF_8)));
+ firehoseService.flush("111111111111", "us-east-1", "shared");
+
+ verify(s3Service).putObject(eq("floci-firehose-results"), anyString(),
+ argThat(body -> new String(body, StandardCharsets.UTF_8).equals("first\n")),
+ eq(OCTET_STREAM), anyMap(), any(PutObjectOptions.class));
+ verify(s3Service, times(1)).putObject(anyString(), anyString(), any(byte[].class), anyString(),
+ anyMap(), any(PutObjectOptions.class));
+ assertEquals("shared", firehoseService.describeDeliveryStream("222222222222", "eu-west-1", "shared")
+ .getDeliveryStreamName());
+
+ firehoseService.deleteDeliveryStream("111111111111", "us-east-1", "shared");
+ assertEquals("shared", firehoseService.describeDeliveryStream("222222222222", "eu-west-1", "shared")
+ .getDeliveryStreamName());
+ }
+
+ @Test
+ void sameNameIsIndependentForEachAccountRegionDimension() {
+ List<String[]> owners = List.of(
+ new String[] {"111111111111", "us-east-1"},
+ new String[] {"111111111111", "eu-west-1"},
+ new String[] {"222222222222", "us-east-1"},
+ new String[] {"222222222222", "eu-west-1"});
+ for (int i = 0; i < owners.size(); i++) {
+ firehoseService.createDeliveryStream(owners.get(i)[1], owners.get(i)[0], "matrix", null,
+ List.of(), null, null);
+ firehoseService.putRecord(owners.get(i)[0], owners.get(i)[1], "matrix",
+ new Record(("owner-" + i).getBytes(StandardCharsets.UTF_8)));
+ }
+
+ firehoseService.flush("111111111111", "us-east-1", "matrix");
+
+ verify(s3Service).putObject(eq("floci-firehose-results"), anyString(),
+ argThat(body -> new String(body, StandardCharsets.UTF_8).equals("owner-0\n")),
+ eq(OCTET_STREAM), anyMap(), any(PutObjectOptions.class));
+ verify(s3Service, times(1)).putObject(anyString(), anyString(), any(byte[].class), anyString(),
+ anyMap(), any(PutObjectOptions.class));
+ for (String[] owner : owners) {
+ assertEquals("matrix", firehoseService.describeDeliveryStream(owner[0], owner[1], "matrix")
+ .getDeliveryStreamName());
+ }
+ }
+
@Test
void deliversToDefaultBucketWithAwsShapedKey() {
firehoseService.createDeliveryStream("my-stream", null);