Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): reset volatile DynamoDB Streams ESM checkpoints on restart (#2077)

Aniket Shukla847c30a52d29
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
index dc9c3ef36b..0f46f82304 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
@@ -62,12 +62,38 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
public void startPersistedPollers() {
for (EventSourceMapping esm : esmStore.listAll()) {
if (esm.isEnabled() && esm.getEventSourceArn().contains(":dynamodb:")) {
+ discardStaleShardCheckpoints(esm);
startPolling(esm);
}
}
LOG.infov("DynamoDbStreamsEventSourcePoller initialized");
}
+ /**
+ * Discards any shard checkpoints a DynamoDB Streams ESM persisted during a previous run,
+ * before its poller is (re)started at startup.
+ *
+ * <p>A DynamoDB stream is volatile: its record buffer and its {@code AtomicLong} sequence
+ * counter live only in memory (see {@link DynamoDbStreamService}) and are never persisted, so a
+ * restart recreates the stream empty and its sequence numbers start over from
+ * {@code 000000000000000000001}. A {@code shardSequenceNumbers} checkpoint saved during the
+ * previous run therefore points <em>past</em> every record in the new stream epoch: resuming
+ * from it with an {@code AFTER_SEQUENCE_NUMBER} iterator silently skips every freshly written
+ * record — no invoke, no error, no log — until the new sequence numbers climb back above the
+ * stale value. Clearing the checkpoint lets the poller resume from {@code TRIM_HORIZON}, which
+ * matches the volatility of the stream itself. See issue #2076.
+ */
+ private void discardStaleShardCheckpoints(EventSourceMapping esm) {
+ if (esm.getShardSequenceNumbers().isEmpty()) {
+ return;
+ }
+ LOG.infov("DynamoDB Streams ESM {0}: discarding {1} stale shard checkpoint(s) persisted by a "
+ + "previous run (stream sequence numbers reset on restart); resuming from TRIM_HORIZON",
+ esm.getUuid(), esm.getShardSequenceNumbers().size());
+ esm.getShardSequenceNumbers().clear();
+ esmStore.saveForAccount(esm.getAccountId(), esm);
+ }
+
@PreDestroy
void shutdown() {
pollExecutor.shutdownNow();
@@ -105,7 +131,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
}
}
- private void pollAndInvoke(EventSourceMapping esm) {
+ void pollAndInvoke(EventSourceMapping esm) {
if (activePolls.putIfAbsent(esm.getUuid(), Boolean.TRUE) != null) {
return;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
new file mode 100644
index 0000000000..1443a1eead
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
@@ -0,0 +1,134 @@
+package io.github.hectorvent.floci.services.lambda;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;
+import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;
+import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;
+import io.github.hectorvent.floci.services.lambda.model.InvocationType;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
+import io.vertx.core.Vertx;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Regression coverage for issue #2076: a DynamoDB Streams ESM must not resume from a shard
+ * checkpoint persisted by a previous run. The stream and its sequence numbers are volatile
+ * (in-memory only), so after a restart the sequence numbers restart from 1 and a stale checkpoint
+ * would silently skip every new record.
+ */
+class DynamoDbStreamsEventSourcePollerTest {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private static final String ACCOUNT_ID = "000000000000";
+ private static final String STREAM_ARN =
+ "arn:aws:dynamodb:us-east-1:000000000000:table/t/stream/2026-08-01T00:00:00.000";
+ private static final String STALE_CHECKPOINT = "000000000000000000634";
+
+ private DynamoDbStreamsEventSourcePoller poller;
+ private DynamoDbStreamService streamService;
+ private LambdaExecutorService executorService;
+ private LambdaFunctionStore functionStore;
+ private EsmStore esmStore;
+
+ @BeforeEach
+ void setUp() {
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);
+ EmulatorConfig.LambdaServiceConfig lambdaConfig = mock(EmulatorConfig.LambdaServiceConfig.class);
+ when(config.services()).thenReturn(services);
+ when(services.lambda()).thenReturn(lambdaConfig);
+ when(lambdaConfig.pollIntervalMs()).thenReturn(1000L);
+
+ streamService = mock(DynamoDbStreamService.class);
+ executorService = mock(LambdaExecutorService.class);
+ functionStore = mock(LambdaFunctionStore.class);
+ esmStore = new EsmStore(new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, ACCOUNT_ID));
+
+ // A mocked Vertx makes setPeriodic a no-op, so startPolling registers no live timer and
+ // the tests drive pollAndInvoke deterministically.
+ poller = new DynamoDbStreamsEventSourcePoller(
+ mock(Vertx.class), streamService, executorService, functionStore,
+ esmStore, OBJECT_MAPPER, config);
+ }
+
+ private EventSourceMapping persistedStreamsEsmWithStaleCheckpoint() {
+ EventSourceMapping esm = new EventSourceMapping();
+ esm.setUuid("esm-1");
+ esm.setAccountId(ACCOUNT_ID);
+ esm.setRegion("us-east-1");
+ esm.setFunctionName("fn");
+ esm.setEventSourceArn(STREAM_ARN);
+ esm.setBatchSize(10);
+ esm.setEnabled(true);
+ esm.getShardSequenceNumbers().put(DynamoDbStreamService.SHARD_ID, STALE_CHECKPOINT);
+ esmStore.saveForAccount(ACCOUNT_ID, esm);
+ return esm;
+ }
+
+ @Test
+ void startPersistedPollersDiscardsStaleShardCheckpoints() {
+ persistedStreamsEsmWithStaleCheckpoint();
+
+ poller.startPersistedPollers();
+
+ EventSourceMapping reloaded = esmStore.getForAccount(ACCOUNT_ID, "esm-1").orElseThrow();
+ assertTrue(reloaded.getShardSequenceNumbers().isEmpty(),
+ "a checkpoint persisted before restart must be discarded at startup, since the "
+ + "stream's sequence numbers reset to 1 — otherwise every new record is skipped");
+ }
+
+ @Test
+ void pollerDeliversPostRestartRecordAfterStartupCheckpointReset() {
+ persistedStreamsEsmWithStaleCheckpoint();
+
+ // The new stream epoch has a single record at sequence 1.
+ DynamoDbStreamRecord record = new DynamoDbStreamRecord();
+ record.setEventName("INSERT");
+ record.setEventSource("aws:dynamodb");
+ record.setAwsRegion("us-east-1");
+ record.setSequenceNumber("000000000000000000001");
+
+ // Resuming from the stale checkpoint (AFTER_SEQUENCE_NUMBER 634) sees nothing — the bug: the
+ // record's sequence (1) is far below the stale checkpoint, so it is silently skipped.
+ when(streamService.getShardIterator(STREAM_ARN, DynamoDbStreamService.SHARD_ID,
+ "AFTER_SEQUENCE_NUMBER", STALE_CHECKPOINT)).thenReturn("iterator-stale");
+ when(streamService.getRecords("iterator-stale", 10))
+ .thenReturn(new DynamoDbStreamService.GetRecordsResult(List.of(), "iterator-stale"));
+ // Resuming from TRIM_HORIZON — where the fix makes the poller restart — delivers the record.
+ when(streamService.getShardIterator(STREAM_ARN, DynamoDbStreamService.SHARD_ID,
+ "TRIM_HORIZON", null)).thenReturn("iterator-trim");
+ when(streamService.getRecords("iterator-trim", 10))
+ .thenReturn(new DynamoDbStreamService.GetRecordsResult(List.of(record), "iterator-trim"));
+
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("fn");
+ when(functionStore.getForAccount(ACCOUNT_ID, "us-east-1", "fn")).thenReturn(Optional.of(fn));
+ when(executorService.invoke(eq(fn), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult()); // success — no functionError
+
+ // Startup invalidates the stale checkpoint...
+ poller.startPersistedPollers();
+ // ...so the poll resumes from TRIM_HORIZON and delivers the record instead of dropping it.
+ // (Were the checkpoint not cleared, the poll would take the AFTER_SEQUENCE_NUMBER branch,
+ // find nothing, and invoke would never be called — this verify would then fail.)
+ poller.pollAndInvoke(esmStore.getForAccount(ACCOUNT_ID, "esm-1").orElseThrow());
+
+ verify(executorService, timeout(2000))
+ .invoke(eq(fn), any(byte[].class), eq(InvocationType.RequestResponse));
+ }
+}