Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): recover a DynamoDB Streams ESM from a trimmed checkpoint (#3412)
Ovidiu62abde805a07
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.javaindex 2d9ee7732e..21910db8a5 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java@@ -31,6 +31,9 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {private static final Logger LOG = Logger.getLogger(DynamoDbStreamsEventSourcePoller.class);+ /** Raised by the stream when a stored checkpoint has aged out of the retained window. */+ private static final String TRIMMED_DATA_ACCESS_EXCEPTION = "TrimmedDataAccessException";+private final Vertx vertx;private final DynamoDbStreamService streamService;private final LambdaExecutorService executorService;@@ -154,11 +157,28 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {String shardId = DynamoDbStreamService.SHARD_ID;String lastSeq = esm.getShardSequenceNumbers().get(shardId);- String iterator = lastSeq == null- ? streamService.getShardIterator(streamArn, shardId, "TRIM_HORIZON", null)- : streamService.getShardIterator(streamArn, shardId, "AFTER_SEQUENCE_NUMBER", lastSeq);-- DynamoDbStreamService.GetRecordsResult result = streamService.getRecords(iterator, esm.getBatchSize());+ DynamoDbStreamService.GetRecordsResult result;+ try {+ String iterator = lastSeq == null+ ? streamService.getShardIterator(streamArn, shardId, "TRIM_HORIZON", null)+ : streamService.getShardIterator(streamArn, shardId, "AFTER_SEQUENCE_NUMBER", lastSeq);+ result = streamService.getRecords(iterator, esm.getBatchSize());+ } catch (AwsException e) {+ if (!TRIMMED_DATA_ACCESS_EXCEPTION.equals(e.getErrorCode())) {+ throw e;+ }+ // The checkpoint fell outside the retained window, so the cursor it names can+ // never succeed again. Retrying it wedges the ESM permanently: every later+ // write reaches the stream and none is ever delivered. Resume from the oldest+ // record still held instead, which is what AWS does when a consumer is+ // overtaken by the trim horizon. Records written between the lost checkpoint+ // and that record are gone from the stream and are not delivered.+ LOG.warnv("DynamoDB Streams ESM {0}: checkpoint {1} was trimmed, resuming from the "+ + "trim horizon; records between were dropped from the stream",+ esm.getUuid(), lastSeq);+ String horizon = streamService.getShardIterator(streamArn, shardId, "TRIM_HORIZON", null);+ result = streamService.getRecords(horizon, esm.getBatchSize());+ }List<DynamoDbStreamRecord> records = result.records();if (records.isEmpty()) {diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.javaindex d1ffa6d260..289d3d3ba2 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java@@ -332,6 +332,67 @@ class DynamoDbStreamsEventSourcePollerTest {assertEquals("s2", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));}+ /**+ * A checkpoint that has aged out of the retained window names a cursor that can never succeed.+ * Retrying it wedges the ESM for good: later writes keep reaching the stream and none is ever+ * delivered. The poller must resume from the trim horizon instead.+ */+ @Test+ void trimmedCheckpointResumesFromTrimHorizonInsteadOfWedging() {+ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("fn");+ when(functionStore.getForAccount(ACCOUNT_ID, "us-east-1", "fn")).thenReturn(Optional.of(fn));++ when(streamService.getShardIterator(eq(STREAM_ARN), eq(DynamoDbStreamService.SHARD_ID),+ eq("AFTER_SEQUENCE_NUMBER"), any())).thenReturn("it-trimmed");+ when(streamService.getShardIterator(eq(STREAM_ARN), eq(DynamoDbStreamService.SHARD_ID),+ eq("TRIM_HORIZON"), any())).thenReturn("it-horizon");+ when(streamService.getRecords("it-trimmed", 10)).thenThrow(+ new AwsException("TrimmedDataAccessException",+ "The requested sequence number has been trimmed", 400));+ when(streamService.getRecords("it-horizon", 10)).thenReturn(+ new DynamoDbStreamService.GetRecordsResult(+ List.of(ddbRecord("s9", "INSERT", "{\"status\":{\"S\":\"active\"}}")), "it-horizon"));+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(new InvokeResult());++ EsmStore store = mock(EsmStore.class);+ EventSourceMapping esm = filterEsm();+ esm.getShardSequenceNumbers().put(DynamoDbStreamService.SHARD_ID, STALE_CHECKPOINT);++ pollerWith(store).pollAndInvoke(esm);++ ArgumentCaptor<byte[]> payload = ArgumentCaptor.forClass(byte[].class);+ verify(executorService, timeout(2000)).invoke(any(), payload.capture(), eq(InvocationType.RequestResponse));+ assertEquals(1, readRecords(payload.getValue()).size());+ verify(store, timeout(2000)).saveForAccount(eq(ACCOUNT_ID), any());+ assertEquals("s9", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));+ }++ /** Only a trimmed checkpoint resets the cursor; any other stream failure retries the same window. */+ @Test+ void nonTrimmedStreamErrorLeavesTheCheckpointAlone() {+ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("fn");+ when(functionStore.getForAccount(ACCOUNT_ID, "us-east-1", "fn")).thenReturn(Optional.of(fn));++ when(streamService.getShardIterator(eq(STREAM_ARN), eq(DynamoDbStreamService.SHARD_ID),+ anyString(), any())).thenReturn("it");+ when(streamService.getRecords("it", 10)).thenThrow(+ new AwsException("InternalServerError", "boom", 500));++ EsmStore store = mock(EsmStore.class);+ EventSourceMapping esm = filterEsm();+ esm.getShardSequenceNumbers().put(DynamoDbStreamService.SHARD_ID, STALE_CHECKPOINT);++ pollerWith(store).pollAndInvoke(esm);+ awaitPollCompletedViaSecondFetch(pollerWith(store), esm);++ verify(executorService, never()).invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));+ verify(store, never()).saveForAccount(anyString(), any());+ assertEquals(STALE_CHECKPOINT, esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));+ }+private JsonNode readRecords(byte[] payload) {try {return OBJECT_MAPPER.readTree(payload).path("Records");