Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): honor LATEST and stop deleted DynamoDB stream mappings writing back (#4318)

Mario Toffia7334cac52ba5
diff --git a/docs/services/lambda.md b/docs/services/lambda.md
index f36baa3cc8..deef5ab7d4 100644
--- a/docs/services/lambda.md
+++ b/docs/services/lambda.md
@@ -834,6 +834,8 @@ Connect Lambda to SQS, Kinesis, or DynamoDB Streams. Self-managed Apache Kafka e
For DynamoDB Streams mappings, Floci retries failed batches with exponential backoff, honors `MaximumRetryAttempts` and `MaximumRecordAgeInSeconds`, and sends discarded batches to configured SQS, SNS, or S3 `DestinationConfig.OnFailure` destinations. With `BisectBatchOnFunctionError`, a batch that fails with a function error is split in half and retried, narrowing it down to the failing record, which then follows the normal retry and record age rules. Splits do not count as retry attempts, and only function errors split: throttles do not, and a partial batch response retries from the reported record. If the OnFailure destination refuses a discarded batch, Floci keeps the checkpoint and retries the send with backoff instead of advancing past the batch.
+A DynamoDB Streams mapping created with `StartingPosition: LATEST` delivers only records written after it is created, so a stream that is empty at creation delivers everything written later. After a restart it still resumes from the trim horizon, because native stream records are volatile. Deleting or disabling a mapping stops any poll that has not yet invoked the function. An invocation already running completes, but its checkpoint is not saved: a deleted mapping is never recreated and its result is dropped, and a disabled mapping re-reads that batch when it is enabled again.
+
```bash
# SQS trigger
QUEUE_ARN=$(aws sqs get-queue-attributes \
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
index da844fbd06..d31c9482b0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
@@ -285,6 +285,18 @@ public class DynamoDbStreamService {
public record GetRecordsResult(List<DynamoDbStreamRecord> records, String nextShardIterator) {}
+ /** Sequence number of the newest record still retained, or null when the stream holds none. */
+ public String latestSequenceNumber(String streamArn) {
+ ConcurrentLinkedDeque<DynamoDbStreamRecord> deque = records.get(streamArn);
+ if (deque == null) {
+ return null;
+ }
+ synchronized (deque) {
+ DynamoDbStreamRecord last = deque.peekLast();
+ return last == null ? null : last.getSequenceNumber();
+ }
+ }
+
public GetRecordsResult getRecords(String shardIterator, Integer limit) {
String[] parts = decodeIterator(shardIterator);
String streamArn = parts[0];
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
index f0792f9f88..1c0c8d1ba8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
@@ -73,6 +73,8 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
private final ConcurrentHashMap<String, Long> retryNotBefore = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, Integer> bisectLimits = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, PendingFailure> pendingFailures = new ConcurrentHashMap<>();
+ final Set<String> stopped = ConcurrentHashMap.newKeySet();
+ private volatile boolean resetting;
private final ExecutorService pollExecutor = Executors.newCachedThreadPool(r -> {
Thread t = new Thread(r, "dynamodb-streams-esm-poller");
t.setDaemon(true);
@@ -122,9 +124,11 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
public void startPersistedPollers() {
for (EventSourceMapping esm : esmStore.listAll()) {
- if (esm.isEnabled() && esm.getEventSourceArn().contains(":dynamodb:")) {
+ if (esm.getEventSourceArn() != null && esm.getEventSourceArn().contains(":dynamodb:")) {
discardStaleShardCheckpoints(esm);
- startPolling(esm);
+ if (esm.isEnabled()) {
+ startPolling(esm);
+ }
}
}
LOG.infov("DynamoDbStreamsEventSourcePoller initialized");
@@ -155,6 +159,17 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
esmStore.saveForAccount(esm.getAccountId(), esm);
}
+ /** Pins a new LATEST mapping after the stream's newest record, so only later writes are delivered. */
+ public void initializeStartingPosition(EventSourceMapping esm) {
+ if (!"LATEST".equals(esm.getStartingPosition()) || !esm.getShardSequenceNumbers().isEmpty()) {
+ return;
+ }
+ String latest = streamService.latestSequenceNumber(esm.getEventSourceArn());
+ if (latest != null) {
+ esm.getShardSequenceNumbers().put(DynamoDbStreamService.SHARD_ID, latest);
+ }
+ }
+
@PreDestroy
void shutdown() {
pollExecutor.shutdownNow();
@@ -162,17 +177,30 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
timerIds.clear();
}
+ @Override
+ public synchronized void beforeReset() {
+ // Taking the monitor drains an in-flight advanceCheckpoint save; it is released before the wipe.
+ resetting = true;
+ }
+
+ @Override
+ public synchronized void afterReset() {
+ resetting = false;
+ }
+
public void clear() {
timerIds.values().forEach(vertx::cancelTimer);
timerIds.clear();
activePolls.clear();
+ stopped.clear();
retryCounts.clear();
retryNotBefore.clear();
bisectLimits.clear();
pendingFailures.clear();
}
- public void startPolling(EventSourceMapping esm) {
+ public synchronized void startPolling(EventSourceMapping esm) {
+ stopped.remove(esm.getUuid());
if (timerIds.containsKey(esm.getUuid())) {
return;
}
@@ -188,7 +216,16 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
LOG.infov("Started DynamoDB Streams polling for ESM {0} → {1}", uuid, esm.getEventSourceArn());
}
- public void stopPolling(String uuid) {
+ /**
+ * Drops the stop tombstone. Safe once the store delete has run, because exists() in the
+ * synchronized advanceCheckpoint then refuses any write-back on its own.
+ */
+ void mappingDeleted(String uuid) {
+ stopped.remove(uuid);
+ }
+
+ public synchronized void stopPolling(String uuid) {
+ stopped.add(uuid);
Long timerId = timerIds.remove(uuid);
if (timerId != null) {
vertx.cancelTimer(timerId);
@@ -259,6 +296,15 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
if (records.isEmpty()) {
return;
}
+ // ponytail: a stop or reset racing this check does not cancel an in-flight invoke; the stop checks in
+ // advanceCheckpoint and dispose drop its checkpoint and OnFailure send. Only a stop landing
+ // between dispose's check and the send itself still delivers; a lock around the send would close it.
+ boolean enabled = esmStore.getForAccount(esm.getAccountId(), esm.getUuid())
+ .map(EventSourceMapping::isEnabled)
+ .orElse(false);
+ if (resetting || !enabled) {
+ return;
+ }
// Advance to the newest FETCHED record whenever the batch is disposed of, invoked or
// fully filtered out, so filtered-out records are consumed, not re-read forever. Leave
@@ -303,6 +349,11 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
}
throw e;
}
+ if (!exists(esm)) {
+ LOG.debugv("DynamoDB Streams ESM {0} was deleted during the invocation, dropping its result",
+ esm.getUuid());
+ return;
+ }
CheckpointOutcome outcome = invokeResult.getFunctionError() == null
? successfulInvocationCheckpoint(esm, invokeResult, lastSeq, records, matched)
@@ -404,9 +455,14 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
/**
* Sends a discarded batch to the OnFailure destination and checkpoints past it. A refused send
* parks the batch under {@code key}, the checkpoint actually persisted, and retries only the send
- * after a backoff. Returns false when the batch was parked.
+ * after a backoff. Returns false only when the batch was parked; a stopped mapping sends and
+ * parks nothing.
*/
private boolean dispose(EventSourceMapping esm, String shardId, String key, PendingFailure failure, long now) {
+ if (isStopped(esm)) {
+ clearBatchState(key);
+ return true;
+ }
if (sendToOnFailureDestination(esm, shardId, failure.records(), failure.invokeResult(),
failure.invokeCount(), failure.condition())) {
clearBatchState(key);
@@ -698,8 +754,19 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
return item;
}
- private void advanceCheckpoint(EventSourceMapping esm, String shardId, String newestSeq) {
+ private synchronized void advanceCheckpoint(EventSourceMapping esm, String shardId, String newestSeq) {
+ if (isStopped(esm)) {
+ return;
+ }
esm.getShardSequenceNumbers().put(shardId, newestSeq);
esmStore.saveForAccount(esm.getAccountId(), esm);
}
+
+ private boolean isStopped(EventSourceMapping esm) {
+ return resetting || stopped.contains(esm.getUuid()) || !exists(esm);
+ }
+
+ private boolean exists(EventSourceMapping esm) {
+ return esmStore.getForAccount(esm.getAccountId(), esm.getUuid()).isPresent();
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
index 7085e3a41d..5039e41dec 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
@@ -1283,6 +1283,9 @@ public class LambdaService implements ResourceProvider {
esm.setSourceAccessConfigurations(sourceAccessConfigurations);
esm.setLastModified(System.currentTimeMillis());
+ if (eventSourceArn != null && eventSourceArn.contains(":dynamodb:")) {
+ dynamodbStreamsPoller.initializeStartingPosition(esm);
+ }
esmStore.save(esm);
if (enabled) {
startPollingHelper(esm);
@@ -1861,6 +1864,9 @@ public class LambdaService implements ResourceProvider {
EventSourceMapping esm = getEventSourceMapping(uuid); // throws 404 if not found
stopPollingHelper(esm);
esmStore.delete(uuid);
+ if (esm.getEventSourceArn() != null && esm.getEventSourceArn().contains(":dynamodb:")) {
+ dynamodbStreamsPoller.mappingDeleted(uuid);
+ }
LOG.infov("Deleted ESM {0}", uuid);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java
index c92604bde7..9cdd712398 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java
@@ -249,6 +249,26 @@ class DynamoDbStreamServiceTest {
assertEquals("000000000000000000002", secondPage.records().get(0).getSequenceNumber());
}
+ @Test
+ void latestSequenceNumberIsNullForAnUnknownOrEmptyStream() {
+ TableDefinition table = createTestTableWithStream();
+
+ assertNull(service.latestSequenceNumber(TABLE_ARN + "/stream/unknown"));
+ assertNull(service.latestSequenceNumber(table.getStreamArn()));
+ }
+
+ @Test
+ void latestSequenceNumberIsTheNewestRetainedRecord() throws Exception {
+ TableDefinition table = createTestTableWithStream();
+ service.enableStream(table.getTableName(), TABLE_ARN, "NEW_IMAGE", "us-east-1");
+ JsonNode item = mapper.readTree("{\"userId\":{\"S\":\"u1\"}}");
+
+ service.captureEvent(table.getTableName(), "INSERT", null, item, table, "us-east-1");
+ service.captureEvent(table.getTableName(), "MODIFY", item, item, table, "us-east-1");
+
+ assertEquals("000000000000000000002", service.latestSequenceNumber(table.getStreamArn()));
+ }
+
@Test
void requiresASequenceNumberForSequenceBasedIterators() {
var table = createTestTableWithStream();
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
index 96fcb8b9ea..3d1c4f11ac 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
@@ -12,7 +12,10 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;
+import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;
+import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
import io.github.hectorvent.floci.services.eventbridge.EventBridgeService;
import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;
@@ -47,6 +50,8 @@ import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -66,6 +71,7 @@ import static org.mockito.Mockito.never;
import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
/**
@@ -155,6 +161,38 @@ class DynamoDbStreamsEventSourcePollerTest {
+ "stream's sequence numbers reset to 1 — otherwise every new record is skipped");
}
+ @Test
+ void startPersistedPollersDiscardsDisabledMappingCheckpointsWithoutPolling() {
+ EventSourceMapping esm = persistedStreamsEsmWithStaleCheckpoint();
+ esm.setEnabled(false);
+ esmStore.saveForAccount(ACCOUNT_ID, esm);
+ Vertx vertx = mock(Vertx.class);
+ DynamoDbStreamsEventSourcePoller p = new DynamoDbStreamsEventSourcePoller(
+ vertx, streamService, executorService, new LambdaTargetResolver(functionStore, aliasStore),
+ esmStore, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service, clock::get);
+
+ p.startPersistedPollers();
+
+ EventSourceMapping reloaded = esmStore.getForAccount(ACCOUNT_ID, "esm-1").orElseThrow();
+ assertTrue(reloaded.getShardSequenceNumbers().isEmpty(),
+ "a disabled mapping's checkpoint is just as stale after restart, and enabling it later "
+ + "would otherwise skip every new record");
+ verifyNoInteractions(vertx);
+ }
+
+ @Test
+ void startPersistedPollersSkipsADisabledMappingWithoutAnEventSourceArn() {
+ EventSourceMapping kafka = new EventSourceMapping();
+ kafka.setUuid("esm-kafka");
+ kafka.setAccountId(ACCOUNT_ID);
+ kafka.setEnabled(false);
+ esmStore.saveForAccount(ACCOUNT_ID, kafka);
+
+ poller.startPersistedPollers();
+
+ assertTrue(esmStore.getForAccount(ACCOUNT_ID, "esm-kafka").isPresent());
+ }
+
@Test
void pollerDeliversPostRestartRecordAfterStartupCheckpointReset() {
persistedStreamsEsmWithStaleCheckpoint();
@@ -198,6 +236,9 @@ class DynamoDbStreamsEventSourcePollerTest {
// ──────────────────────────── FilterCriteria ────────────────────────────
private DynamoDbStreamsEventSourcePoller pollerWith(EsmStore store) {
+ // The mock records writes for verification; reads see what filterEsm stored, as a real store would.
+ when(store.getForAccount(anyString(), anyString()))
+ .thenAnswer(inv -> esmStore.getForAccount(inv.getArgument(0), inv.getArgument(1)));
return new DynamoDbStreamsEventSourcePoller(
mock(Vertx.class), streamService, executorService, new LambdaTargetResolver(functionStore, aliasStore),
store, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service, clock::get);
@@ -252,6 +293,7 @@ class DynamoDbStreamsEventSourcePollerTest {
fc.setFilters(filters);
esm.setFilterCriteria(fc);
}
+ esmStore.saveForAccount(ACCOUNT_ID, esm);
return esm;
}
@@ -1619,6 +1661,403 @@ class DynamoDbStreamsEventSourcePollerTest {
assertEquals("s1", checkpoint(esm));
}
+ // ──────────────────── StartingPosition and mapping lifecycle (#4311) ────────────────────
+
+ private static final String TABLE_ARN = "arn:aws:dynamodb:us-east-1:000000000000:table/t";
+
+ private DynamoDbStreamService realStream() {
+ DynamoDbStreamService streams = new DynamoDbStreamService(OBJECT_MAPPER, mock(StorageFactory.class));
+ streams.enableStream("t", TABLE_ARN, "NEW_AND_OLD_IMAGES", "us-east-1", STREAM_ARN);
+ return streams;
+ }
+
+ private void write(DynamoDbStreamService streams, String pk) {
+ TableDefinition table = new TableDefinition("t", List.of(new KeySchemaElement("pk", "HASH")),
+ List.of(new AttributeDefinition("pk", "S")), "us-east-1", ACCOUNT_ID);
+ ObjectNode item = OBJECT_MAPPER.createObjectNode();
+ item.putObject("pk").put("S", pk);
+ streams.captureEvent("t", "INSERT", null, item, table, "us-east-1");
+ }
+
+ private DynamoDbStreamsEventSourcePoller pollerOver(DynamoDbStreamService streams) {
+ return new DynamoDbStreamsEventSourcePoller(
+ mock(Vertx.class), streams, executorService, new LambdaTargetResolver(functionStore, aliasStore),
+ esmStore, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service, clock::get);
+ }
+
+ private EventSourceMapping streamsEsm(String uuid, String startingPosition) {
+ EventSourceMapping esm = new EventSourceMapping();
+ esm.setUuid(uuid);
+ esm.setAccountId(ACCOUNT_ID);
+ esm.setRegion("us-east-1");
+ esm.setFunctionName("fn");
+ esm.setEventSourceArn(STREAM_ARN);
+ esm.setBatchSize(10);
+ esm.setEnabled(true);
+ esm.setStartingPosition(startingPosition);
+ return esm;
+ }
+
+ private void stubFunctionSucceeds() {
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("fn");
+ when(functionStore.getForAccount(ACCOUNT_ID, "us-east-1", "fn")).thenReturn(Optional.of(fn));
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult());
+ }
+
+ private List<String> deliveredKeys(byte[] payload) {
+ List<String> keys = new ArrayList<>();
+ for (JsonNode rec : readRecords(payload)) {
+ keys.add(rec.path("dynamodb").path("Keys").path("pk").path("S").asText());
+ }
+ return keys;
+ }
+
+ /** One-record read that signals {@code reading} and then holds until {@code release} opens. */
+ private void stubBlockingRead(CountDownLatch reading, CountDownLatch release) {
+ stubTrimHorizon(List.of());
+ when(streamService.getRecords("it", 10)).thenAnswer(inv -> {
+ reading.countDown();
+ release.await(5, TimeUnit.SECONDS);
+ return new DynamoDbStreamService.GetRecordsResult(List.of(ddbRecord("s1", "INSERT", "{}")), "it");
+ });
+ }
+
+ /** Invocation that signals {@code invoking} and then holds until {@code release} opens. */
+ private void stubBlockingInvoke(CountDownLatch invoking, CountDownLatch release, InvokeResult result) {
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenAnswer(inv -> {
+ invoking.countDown();
+ release.await(5, TimeUnit.SECONDS);
+ return result;
+ });
+ }
+
+ @Test
+ void latestMappingDeliversOnlyRecordsWrittenAfterItWasCreated() {
+ DynamoDbStreamService streams = realStream();
+ write(streams, "old-1");
+ write(streams, "old-2");
+ stubFunctionSucceeds();
+ DynamoDbStreamsEventSourcePoller p = pollerOver(streams);
+ EventSourceMapping esm = streamsEsm("esm-latest", "LATEST");
+ p.initializeStartingPosition(esm);
+ esmStore.saveForAccount(ACCOUNT_ID, esm);
+
+ write(streams, "new");
+ p.pollAndInvoke(esm);
+
+ ArgumentCaptor<byte[]> payload = ArgumentCaptor.forClass(byte[].class);
+ verify(executorService, timeout(2000)).invoke(any(), payload.capture(), eq(InvocationType.RequestResponse));
+ assertEquals(List.of("new"), deliveredKeys(payload.getValue()));
+ }
+
+ @Test
+ void latestMappingOnAnEmptyStreamDeliversEveryLaterRecord() throws Exception {
+ DynamoDbStreamService streams = realStream();
+ stubFunctionSucceeds();
+ DynamoDbStreamsEventSourcePoller p = pollerOver(streams);
+ EventSourceMapping esm = streamsEsm("esm-latest", "LATEST");
+ p.initializeStartingPosition(esm);
+ esmStore.saveForAccount(ACCOUNT_ID, esm);
+ assertTrue(esm.getShardSequenceNumbers().isEmpty(), "an empty stream has no record to start after");
+
+ p.pollAndInvoke(esm);
+ awaitPollCompleted(p);
+ assertTrue(esm.getShardSequenceNumbers().isEmpty(), "an empty poll must not pin the cursor");
+ verify(executorService, never()).invoke(any(), any(byte[].class), any());
+
+ write(streams, "first");
+ p.pollAndInvoke(esm);
+
+ ArgumentCaptor<byte[]> payload = ArgumentCaptor.forClass(byte[].class);
+ verify(executorService, timeout(2000)).invoke(any(), payload.capture(), eq(InvocationType.RequestResponse));
+ assertEquals(List.of("first"), deliveredKeys(payload.getValue()));
+ }
+
+ @Test
+ void trimHorizonMappingIsNotPinnedAtCreation() {
+ DynamoDbStreamService streams = realStream();
+ write(streams, "old");
+ EventSourceMapping esm = streamsEsm("esm-trim", "TRIM_HORIZON");
+
+ pollerOver(streams).initializeStartingPosition(esm);
+
+ assertTrue(esm.getShardSequenceNumbers().isEmpty());
+ }
+
+ @Test
+ void reEnabledMappingResumesAfterItsLastProcessedRecord() throws Exception {
+ DynamoDbStreamService streams = realStream();
+ write(streams, "old");
+ stubFunctionSucceeds();
+ DynamoDbStreamsEventSourcePoller p = pollerOver(streams);
+ EventSourceMapping esm = streamsEsm("esm-latest", "LATEST");
+ p.initializeStartingPosition(esm);
+ esmStore.saveForAccount(ACCOUNT_ID, esm);
+ write(streams, "a");
+ p.pollAndInvoke(esm);
+ verify(executorService, timeout(2000)).invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));
+ awaitPollCompleted(p);
+
+ esm.setEnabled(false);
+ esmStore.saveForAccount(ACCOUNT_ID, esm);
+ p.stopPolling(esm.getUuid());
+ write(streams, "b");
+ esm.setEnabled(true);
+ esmStore.saveForAccount(ACCOUNT_ID, esm);
+ p.startPolling(esm);
+ p.pollAndInvoke(esm);
+
+ ArgumentCaptor<byte[]> payload = ArgumentCaptor.forClass(byte[].class);
+ verify(executorService, timeout(2000).times(2))
+ .invoke(any(), payload.capture(), eq(InvocationType.RequestResponse));
+ assertEquals(List.of("a"), deliveredKeys(payload.getAllValues().get(0)));
+ assertEquals(List.of("b"), deliveredKeys(payload.getAllValues().get(1)),
+ "re-enabling resumes after the last processed record, neither from the trim horizon nor LATEST");
+ }
+
+ @Test
+ void mappingDeletedDuringABlockedReadIsNotInvokedOrResurrected() throws Exception {
+ CountDownLatch reading = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ stubBlockingRead(reading, release);
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult());
+ EventSourceMapping esm = streamsEsm("esm-live", null);
+ esmStore.saveForAccount(ACCOUNT_ID, esm);
+
+ poller.pollAndInvoke(esm);
+ assertTrue(reading.await(2, TimeUnit.SECONDS));
+ poller.stopPolling(esm.getUuid());
+ esmStore.delete(esm.getUuid());
+ release.countDown();
+ awaitPollCompleted(poller);
+
+ verify(executorService, never()).invoke(any(), any(byte[].class), any());
+ assertTrue(esmStore.getForAccount(ACCOUNT_ID, esm.getUuid()).isEmpty(),
+ "a deleted mapping must not reappear in List/Get");
+ }
+
+ @Test
+ void mappingDisabledDuringABlockedReadIsNotInvoked() throws Exception {
+ CountDownLatch reading = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ stubBlockingRead(reading, release);
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult());
+ EventSourceMapping esm = streamsEsm("esm-live", null);
+ esmStore.saveForAccount(ACCOUNT_ID, esm);
+
+ poller.pollAndInvoke(esm);
+ assertTrue(reading.await(2, TimeUnit.SECONDS));
+ esm.setEnabled(false);
+ esmStore.saveForAccount(ACCOUNT_ID, esm);
+ poller.stopPolling(esm.getUuid());
+ release.countDown();
+ awaitPollCompleted(poller);
+
+ verify(executorService, never()).invoke(any(), any(byte[].class), any());
+ }
+
+ @Test
+ void mappingDeletedDuringASuccessfulInvokeDropsTheResult() throws Exception {
+ assertDeletionDuringInvokeDropsTheResult(new InvokeResult());
+ }
+
+ @Test
+ void mappingDeletedDuringAFailedInvokeSendsNothingToItsDestination() throws Exception {
+ InvokeResult error = new InvokeResult();
+ error.setFunctionError("Unhandled");
+ assertDeletionDuringInvokeDropsTheResult(error);
+ }
+
+ private void assertDeletionDuringInvokeDropsTheResult(InvokeResult result) throws Exception {
+ stubTrimHorizon(List.of(ddbRecord("s1", "INSERT", "{}")));
+ CountDownLatch invoking = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ stubBlockingInvoke(invoking, release, result);
+ EventSourceMapping esm = streamsEsm("esm-live", null);
+ esm.setMaximumRetryAttempts(0);
+ EventSourceMapping.OnFailure onFailure = new EventSourceMapping.OnFailure();
+ onFailure.setDestination("arn:aws:sqs:us-east-1:000000000000:my-dlq");
+ EventSourceMapping.DestinationConfig destinationConfig = new EventSourceMapping.DestinationConfig();
+ destinationConfig.setOnFailure(onFailure);
+ esm.setDestinationConfig(destinationConfig);
+ esmStore.saveForAccount(ACCOUNT_ID, esm);
+
+ poller.pollAndInvoke(esm);
+ assertTrue(invoking.await(2, TimeUnit.SECONDS));
+ poller.stopPolling(esm.getUuid());
+ esmStore.delete(esm.getUuid());
+ release.countDown();
+ awaitPollCompleted(poller);
+
+ assertTrue(esmStore.getForAccount(ACCOUNT_ID, esm.getUuid()).isEmpty(),
+ "a deleted mapping must not reappear in List/Get");
+ assertTrue(esm.getShardSequenceNumbers().isEmpty(), "no checkpoint is written for a deleted mapping");
+ verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString());
+ }
+
+ @Test
+ void mappingDeletedDuringABisectingFailedInvokeLeavesNoBisectStateOrDestinationSend() throws Exception {
+ stubStream("s1", "s2", "s3", "s4");
+ EventSourceMapping esm = esmWithDlq(0);
+ esm.setBisectBatchOnFunctionError(true);
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+ List<List<String>> invocations = recordInvocations(seqs -> {
+ p.stopPolling(esm.getUuid());
+ esmStore.delete(esm.getUuid());
+ InvokeResult error = new InvokeResult();
+ error.setFunctionError("Unhandled");
+ return error;
+ });
+
+ pollOnce(p, esm);
+ pollOnce(p, esm);
+
+ assertEquals(List.of(List.of("s1", "s2", "s3", "s4")), invocations);
+ // A recorded bisect would halve the next fetch; both fetches must still use the full batch size.
+ verify(streamService, times(2)).getRecords(anyString(), eq(10));
+ assertNull(checkpoint(esm));
+ verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString());
+ }
+
+ @Test
+ void parkedFailureIsNotResentOnceTheMappingIsStopped() throws Exception {
+ stubStream("s1");
+ List<List<String>> invocations = failInvocationsContaining("s1");
+ refuseSqsSends(1);
+ EventSourceMapping esm = esmWithDlq(0);
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+ pollOnce(p, esm);
+ assertNull(checkpoint(esm));
+ // A worker that loaded the parked entry before stopPolling cleared it sees only the tombstone.
+ p.stopped.add(esm.getUuid());
+
+ advancePastRetry(p);
+ pollOnce(p, esm);
+
+ verify(sqsService, times(1)).sendMessage(anyString(), anyString(), anyInt(), anyString());
+ assertNull(checkpoint(esm));
+ assertEquals(List.of(List.of("s1")), invocations);
+ }
+
+ @Test
+ void checkpointIsNotSavedWhileAResetIsInProgress() throws Exception {
+ stubStream("s1");
+ EventSourceMapping esm = filterEsm();
+ EsmStore store = mock(EsmStore.class);
+ DynamoDbStreamsEventSourcePoller p = pollerWith(store);
+ AtomicBoolean resetDuringInvoke = new AtomicBoolean(true);
+ List<List<String>> invocations = recordInvocations(seqs -> {
+ if (resetDuringInvoke.getAndSet(false)) {
+ p.beforeReset();
+ }
+ return new InvokeResult();
+ });
+
+ pollOnce(p, esm);
+ assertNull(checkpoint(esm));
+ verify(store, never()).saveForAccount(anyString(), any());
+
+ p.afterReset();
+ pollOnce(p, esm);
+
+ assertEquals(List.of(List.of("s1"), List.of("s1")), invocations);
+ assertEquals("s1", checkpoint(esm));
+ verify(store).saveForAccount(eq(ACCOUNT_ID), any());
+ }
+
+ @Test
+ void pollDuringAResetDoesNotInvokeUntilTheResetEnds() throws Exception {
+ stubStream("s1");
+ List<List<String>> invocations = recordInvocations(seqs -> new InvokeResult());
+ EventSourceMapping esm = filterEsm();
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+
+ p.beforeReset();
+ pollOnce(p, esm);
+ assertTrue(invocations.isEmpty(), "a poll during a reset must not invoke the function");
+
+ p.afterReset();
+ pollOnce(p, esm);
+
+ assertEquals(List.of(List.of("s1")), invocations);
+ assertEquals("s1", checkpoint(esm));
+ }
+
+ @Test
+ void stopPollingDuringAnInvocationBlocksItsCheckpointWrite() throws Exception {
+ stubTrimHorizon(List.of(ddbRecord("s1", "INSERT", "{}")));
+ EsmStore store = mock(EsmStore.class);
+ EventSourceMapping esm = filterEsm();
+ DynamoDbStreamsEventSourcePoller p = pollerWith(store);
+ // A delete has stopped the poller but has not reached the store yet, so the mapping still exists.
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenAnswer(inv -> {
+ p.stopPolling(esm.getUuid());
+ return new InvokeResult();
+ });
+
+ p.pollAndInvoke(esm);
+ verify(executorService, timeout(2000)).invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));
+ awaitPollCompleted(p);
+
+ verify(store, never()).saveForAccount(anyString(), any());
+ }
+
+ @Test
+ void startPollingAfterStopPollingLetsTheCheckpointAdvanceAgain() {
+ stubTrimHorizon(List.of(ddbRecord("s1", "INSERT", "{}")));
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult());
+ EsmStore store = mock(EsmStore.class);
+ EventSourceMapping esm = filterEsm();
+ DynamoDbStreamsEventSourcePoller p = pollerWith(store);
+
+ p.stopPolling(esm.getUuid());
+ p.startPolling(esm);
+ p.pollAndInvoke(esm);
+
+ verify(store, timeout(2000)).saveForAccount(eq(ACCOUNT_ID), any());
+ assertEquals("s1", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));
+ }
+
+ @Test
+ void mappingDeletedReleasesItsStopTombstone() {
+ poller.stopPolling("esm-gone");
+
+ poller.mappingDeleted("esm-gone");
+
+ assertTrue(poller.stopped.isEmpty());
+ }
+
+ @Test
+ void resetMappingsInFlightInvokeDoesNotResurrectIt() throws Exception {
+ stubTrimHorizon(List.of(ddbRecord("s1", "INSERT", "{}")));
+ CountDownLatch invoking = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ stubBlockingInvoke(invoking, release, new InvokeResult());
+ EventSourceMapping old = streamsEsm("esm-old", null);
+ esmStore.saveForAccount(ACCOUNT_ID, old);
+
+ poller.pollAndInvoke(old);
+ assertTrue(invoking.await(2, TimeUnit.SECONDS));
+ esmStore.delete(old.getUuid());
+ poller.clear();
+ EventSourceMapping recreated = streamsEsm("esm-new", null);
+ esmStore.saveForAccount(ACCOUNT_ID, recreated);
+ // clear() forgot the in-flight poll; re-mark it so awaitPollCompleted waits for the old worker.
+ poller.activePolls.put(old.getUuid(), Boolean.TRUE);
+ release.countDown();
+ awaitPollCompleted(poller);
+
+ assertTrue(esmStore.getForAccount(ACCOUNT_ID, old.getUuid()).isEmpty(),
+ "a mapping wiped by a reset must not be written back by its in-flight poll");
+ }
+
private void awaitPollCompleted(DynamoDbStreamsEventSourcePoller poller) throws InterruptedException {
long deadline = System.currentTimeMillis() + 3_000;
while (!poller.activePolls.isEmpty() && System.currentTimeMillis() < deadline) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java
index 33452663d0..50de9f1e13 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java
@@ -1,13 +1,19 @@
package io.github.hectorvent.floci.services.lambda;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;
import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.path.json.JsonPath;
import io.restassured.path.json.config.JsonPathConfig;
+import jakarta.inject.Inject;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.UUID;
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.*;
@@ -45,6 +51,9 @@ class EsmIntegrationTest {
private static String esmUuid;
+ @Inject
+ EsmStore esmStore;
+
@Test
@Order(1)
void setupSqsQueue() {
@@ -257,6 +266,68 @@ class EsmIntegrationTest {
.statusCode(202);
}
+ @Test
+ void createLatestDynamoDbMappingStartsAfterTheNewestRecord() {
+ String tableName = "esm-latest-" + UUID.randomUUID();
+ String streamArn = dynamoDb("DynamoDB_20120810.CreateTable", """
+ {
+ "TableName": "%s",
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}
+ }
+ """.formatted(tableName)).getString("TableDescription.LatestStreamArn");
+ for (String pk : List.of("before-1", "before-2")) {
+ dynamoDb("DynamoDB_20120810.PutItem", """
+ {"TableName": "%s", "Item": {"pk": {"S": "%s"}}}
+ """.formatted(tableName, pk));
+ }
+ String iterator = dynamoDb("DynamoDBStreams_20120810.GetShardIterator", """
+ {"StreamArn": "%s", "ShardId": "%s", "ShardIteratorType": "TRIM_HORIZON"}
+ """.formatted(streamArn, DynamoDbStreamService.SHARD_ID)).getString("ShardIterator");
+ String newestSequence = dynamoDb("DynamoDBStreams_20120810.GetRecords", """
+ {"ShardIterator": "%s"}
+ """.formatted(iterator)).getString("Records[-1].dynamodb.SequenceNumber");
+
+ // Disabled so nothing polls: only the starting point captured at creation is under test.
+ String uuid = given()
+ .contentType("application/json")
+ .body("""
+ {
+ "FunctionName": "%s",
+ "EventSourceArn": "%s",
+ "StartingPosition": "LATEST",
+ "Enabled": false
+ }
+ """.formatted(FUNCTION_NAME, streamArn))
+ .when()
+ .post(LAMBDA_BASE + "/event-source-mappings/")
+ .then()
+ .statusCode(202)
+ .extract()
+ .path("UUID");
+
+ assertEquals(newestSequence, esmStore.getForAccount(ACCOUNT_ID, uuid).orElseThrow()
+ .getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));
+
+ given().delete(LAMBDA_BASE + "/event-source-mappings/" + uuid).then().statusCode(202);
+ dynamoDb("DynamoDB_20120810.DeleteTable", "{\"TableName\": \"%s\"}".formatted(tableName));
+ }
+
+ private static JsonPath dynamoDb(String target, String body) {
+ return given()
+ .header("X-Amz-Target", target)
+ .contentType("application/x-amz-json-1.0")
+ .body(body.getBytes(StandardCharsets.UTF_8))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract()
+ .jsonPath();
+ }
+
@Test
void createEventSourceMappingRejectsMaximumRecordAgeBelowMinimum() {
given()