Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): recover a DynamoDB Streams ESM from a trimmed checkpoint (#3412)

Ovidiu62abde805a07
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
index 2d9ee7732e..21910db8a5 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
@@ -31,6 +31,9 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
private static final Logger LOG = Logger.getLogger(DynamoDbStreamsEventSourcePoller.class);
+ /** Raised by the stream when a stored checkpoint has aged out of the retained window. */
+ private static final String TRIMMED_DATA_ACCESS_EXCEPTION = "TrimmedDataAccessException";
+
private final Vertx vertx;
private final DynamoDbStreamService streamService;
private final LambdaExecutorService executorService;
@@ -154,11 +157,28 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
String shardId = DynamoDbStreamService.SHARD_ID;
String lastSeq = esm.getShardSequenceNumbers().get(shardId);
- String iterator = lastSeq == null
- ? streamService.getShardIterator(streamArn, shardId, "TRIM_HORIZON", null)
- : streamService.getShardIterator(streamArn, shardId, "AFTER_SEQUENCE_NUMBER", lastSeq);
-
- DynamoDbStreamService.GetRecordsResult result = streamService.getRecords(iterator, esm.getBatchSize());
+ DynamoDbStreamService.GetRecordsResult result;
+ try {
+ String iterator = lastSeq == null
+ ? streamService.getShardIterator(streamArn, shardId, "TRIM_HORIZON", null)
+ : streamService.getShardIterator(streamArn, shardId, "AFTER_SEQUENCE_NUMBER", lastSeq);
+ result = streamService.getRecords(iterator, esm.getBatchSize());
+ } catch (AwsException e) {
+ if (!TRIMMED_DATA_ACCESS_EXCEPTION.equals(e.getErrorCode())) {
+ throw e;
+ }
+ // The checkpoint fell outside the retained window, so the cursor it names can
+ // never succeed again. Retrying it wedges the ESM permanently: every later
+ // write reaches the stream and none is ever delivered. Resume from the oldest
+ // record still held instead, which is what AWS does when a consumer is
+ // overtaken by the trim horizon. Records written between the lost checkpoint
+ // and that record are gone from the stream and are not delivered.
+ LOG.warnv("DynamoDB Streams ESM {0}: checkpoint {1} was trimmed, resuming from the "
+ + "trim horizon; records between were dropped from the stream",
+ esm.getUuid(), lastSeq);
+ String horizon = streamService.getShardIterator(streamArn, shardId, "TRIM_HORIZON", null);
+ result = streamService.getRecords(horizon, esm.getBatchSize());
+ }
List<DynamoDbStreamRecord> records = result.records();
if (records.isEmpty()) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
index d1ffa6d260..289d3d3ba2 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
@@ -332,6 +332,67 @@ class DynamoDbStreamsEventSourcePollerTest {
assertEquals("s2", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));
}
+ /**
+ * A checkpoint that has aged out of the retained window names a cursor that can never succeed.
+ * Retrying it wedges the ESM for good: later writes keep reaching the stream and none is ever
+ * delivered. The poller must resume from the trim horizon instead.
+ */
+ @Test
+ void trimmedCheckpointResumesFromTrimHorizonInsteadOfWedging() {
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("fn");
+ when(functionStore.getForAccount(ACCOUNT_ID, "us-east-1", "fn")).thenReturn(Optional.of(fn));
+
+ when(streamService.getShardIterator(eq(STREAM_ARN), eq(DynamoDbStreamService.SHARD_ID),
+ eq("AFTER_SEQUENCE_NUMBER"), any())).thenReturn("it-trimmed");
+ when(streamService.getShardIterator(eq(STREAM_ARN), eq(DynamoDbStreamService.SHARD_ID),
+ eq("TRIM_HORIZON"), any())).thenReturn("it-horizon");
+ when(streamService.getRecords("it-trimmed", 10)).thenThrow(
+ new AwsException("TrimmedDataAccessException",
+ "The requested sequence number has been trimmed", 400));
+ when(streamService.getRecords("it-horizon", 10)).thenReturn(
+ new DynamoDbStreamService.GetRecordsResult(
+ List.of(ddbRecord("s9", "INSERT", "{\"status\":{\"S\":\"active\"}}")), "it-horizon"));
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult());
+
+ EsmStore store = mock(EsmStore.class);
+ EventSourceMapping esm = filterEsm();
+ esm.getShardSequenceNumbers().put(DynamoDbStreamService.SHARD_ID, STALE_CHECKPOINT);
+
+ pollerWith(store).pollAndInvoke(esm);
+
+ ArgumentCaptor<byte[]> payload = ArgumentCaptor.forClass(byte[].class);
+ verify(executorService, timeout(2000)).invoke(any(), payload.capture(), eq(InvocationType.RequestResponse));
+ assertEquals(1, readRecords(payload.getValue()).size());
+ verify(store, timeout(2000)).saveForAccount(eq(ACCOUNT_ID), any());
+ assertEquals("s9", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));
+ }
+
+ /** Only a trimmed checkpoint resets the cursor; any other stream failure retries the same window. */
+ @Test
+ void nonTrimmedStreamErrorLeavesTheCheckpointAlone() {
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("fn");
+ when(functionStore.getForAccount(ACCOUNT_ID, "us-east-1", "fn")).thenReturn(Optional.of(fn));
+
+ when(streamService.getShardIterator(eq(STREAM_ARN), eq(DynamoDbStreamService.SHARD_ID),
+ anyString(), any())).thenReturn("it");
+ when(streamService.getRecords("it", 10)).thenThrow(
+ new AwsException("InternalServerError", "boom", 500));
+
+ EsmStore store = mock(EsmStore.class);
+ EventSourceMapping esm = filterEsm();
+ esm.getShardSequenceNumbers().put(DynamoDbStreamService.SHARD_ID, STALE_CHECKPOINT);
+
+ pollerWith(store).pollAndInvoke(esm);
+ awaitPollCompletedViaSecondFetch(pollerWith(store), esm);
+
+ verify(executorService, never()).invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));
+ verify(store, never()).saveForAccount(anyString(), any());
+ assertEquals(STALE_CHECKPOINT, esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));
+ }
+
private JsonNode readRecords(byte[] payload) {
try {
return OBJECT_MAPPER.readTree(payload).path("Records");