Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): honor DynamoDB stream partial batch failures (#3466)
XuJianc6316d10cffa
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.javaindex 21910db8a5..2d048d76c9 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java@@ -21,7 +21,11 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.util.ArrayList;+import java.util.HashMap;+import java.util.HashSet;import java.util.List;+import java.util.Map;+import java.util.Set;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;@@ -223,7 +227,11 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {}if (invokeResult.getFunctionError() == null) {- advanceCheckpoint(esm, shardId, newestFetchedSeq);+ String checkpoint = successfulInvocationCheckpoint(+ esm, invokeResult, lastSeq, records, matched);+ if (checkpoint != null && !checkpoint.equals(lastSeq)) {+ advanceCheckpoint(esm, shardId, checkpoint);+ }} else {LOG.warnv("DynamoDB Streams ESM {0}: Lambda returned error [{1}], records will be retried",esm.getUuid(), invokeResult.getFunctionError());@@ -236,6 +244,75 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {});}+ /**+ * Returns the last record that can be consumed after a successful invocation. Floci stores the+ * last consumed sequence and resumes with {@code AFTER_SEQUENCE_NUMBER}, so a partial failure+ * checkpoints the record immediately before AWS's lowest reported failed sequence.+ */+ private String successfulInvocationCheckpoint(EventSourceMapping esm, InvokeResult invokeResult,+ String previousCheckpoint,+ List<DynamoDbStreamRecord> fetched,+ List<DynamoDbStreamRecord> delivered) {+ String newestFetchedSeq = fetched.get(fetched.size() - 1).getSequenceNumber();+ byte[] payload = invokeResult.getPayload();+ if (!esm.isReportBatchItemFailures() || payload == null || payload.length == 0) {+ return newestFetchedSeq;+ }++ try {+ JsonNode response = objectMapper.readTree(payload);+ JsonNode failures = response.get("batchItemFailures");+ if (failures == null || failures.isNull()) {+ return newestFetchedSeq;+ }+ if (!failures.isArray()) {+ return retryWholeBatch(esm, previousCheckpoint, "batchItemFailures is not an array");+ }++ Map<String, Integer> fetchedIndexes = new HashMap<>();+ for (int i = 0; i < fetched.size(); i++) {+ fetchedIndexes.put(fetched.get(i).getSequenceNumber(), i);+ }+ Set<String> deliveredSequences = new HashSet<>();+ for (DynamoDbStreamRecord record : delivered) {+ deliveredSequences.add(record.getSequenceNumber());+ }++ int lowestFailedIndex = fetched.size();+ for (JsonNode item : failures) {+ JsonNode identifier = item.get("itemIdentifier");+ if (identifier == null || identifier.isNull() || identifier.asText().isEmpty()) {+ return retryWholeBatch(esm, previousCheckpoint,+ "entry has a missing, null or empty itemIdentifier");+ }+ String sequenceNumber = identifier.asText();+ Integer index = fetchedIndexes.get(sequenceNumber);+ if (index == null || !deliveredSequences.contains(sequenceNumber)) {+ return retryWholeBatch(esm, previousCheckpoint,+ "itemIdentifier " + sequenceNumber + " is not in the delivered batch");+ }+ lowestFailedIndex = Math.min(lowestFailedIndex, index);+ }++ if (lowestFailedIndex == fetched.size()) {+ return newestFetchedSeq;+ }+ return lowestFailedIndex == 0+ ? previousCheckpoint+ : fetched.get(lowestFailedIndex - 1).getSequenceNumber();+ } catch (Exception e) {+ return retryWholeBatch(esm, previousCheckpoint,+ "response is not valid JSON: " + e.getMessage());+ }+ }++ private String retryWholeBatch(EventSourceMapping esm, String previousCheckpoint, String reason) {+ LOG.warnv("DynamoDB Streams ESM {0}: malformed batchItemFailures response ({1}), "+ + "retrying the whole batch",+ esm.getUuid(), reason);+ return previousCheckpoint;+ }+private String buildDynamoDbEvent(List<DynamoDbStreamRecord> records, EventSourceMapping esm) {try {ObjectNode root = objectMapper.createObjectNode();diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.javaindex 289d3d3ba2..44ab2ecf9d 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java@@ -332,6 +332,91 @@ class DynamoDbStreamsEventSourcePollerTest {assertEquals("s2", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));}+ @Test+ void partialBatchFailureCheckpointsBeforeLowestFailedRecord() {+ stubTrimHorizon(List.of(+ ddbRecord("s1", "INSERT", "{}"),+ ddbRecord("s2", "INSERT", "{}"),+ ddbRecord("s3", "INSERT", "{}")));+ InvokeResult result = new InvokeResult();+ result.setPayload("{\"batchItemFailures\":[{\"itemIdentifier\":\"s3\"},{\"itemIdentifier\":\"s2\"}]}".getBytes());+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(result);+ EsmStore store = mock(EsmStore.class);+ EventSourceMapping esm = filterEsm();+ esm.setFunctionResponseTypes(List.of("ReportBatchItemFailures"));++ pollerWith(store).pollAndInvoke(esm);++ verify(store, timeout(2000)).saveForAccount(eq(ACCOUNT_ID), any());+ assertEquals("s1", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));+ }++ @Test+ void partialBatchFailureAtFirstRecordKeepsPriorCheckpoint() {+ stubTrimHorizon(List.of(+ ddbRecord("s1", "INSERT", "{}"),+ ddbRecord("s2", "INSERT", "{}")));+ InvokeResult result = new InvokeResult();+ result.setPayload("{\"batchItemFailures\":[{\"itemIdentifier\":\"s1\"}]}".getBytes());+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(result);+ EsmStore store = mock(EsmStore.class);+ EventSourceMapping esm = filterEsm();+ esm.setFunctionResponseTypes(List.of("ReportBatchItemFailures"));+ esm.getShardSequenceNumbers().put(DynamoDbStreamService.SHARD_ID, "s0");+ DynamoDbStreamsEventSourcePoller p = pollerWith(store);++ p.pollAndInvoke(esm);+ verify(executorService, timeout(2000))+ .invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));+ awaitPollCompletedViaSecondFetch(p, esm);++ verify(store, never()).saveForAccount(anyString(), any());+ assertEquals("s0", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));+ }++ @Test+ void malformedPartialBatchResponseRetriesWholeBatch() {+ stubTrimHorizon(List.of(+ ddbRecord("s1", "INSERT", "{}"),+ ddbRecord("s2", "INSERT", "{}")));+ InvokeResult result = new InvokeResult();+ result.setPayload("{\"batchItemFailures\":[{\"itemIdentifier\":\"unknown\"}]}".getBytes());+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(result);+ EsmStore store = mock(EsmStore.class);+ EventSourceMapping esm = filterEsm();+ esm.setFunctionResponseTypes(List.of("ReportBatchItemFailures"));+ DynamoDbStreamsEventSourcePoller p = pollerWith(store);++ p.pollAndInvoke(esm);+ verify(executorService, timeout(2000))+ .invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));+ awaitPollCompletedViaSecondFetch(p, esm);++ verify(store, never()).saveForAccount(anyString(), any());+ assertNull(esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));+ }++ @Test+ void partialBatchResponseIsIgnoredWhenNotConfigured() {+ stubTrimHorizon(List.of(+ ddbRecord("s1", "INSERT", "{}"),+ ddbRecord("s2", "INSERT", "{}")));+ InvokeResult result = new InvokeResult();+ result.setPayload("{\"batchItemFailures\":[{\"itemIdentifier\":\"s1\"}]}".getBytes());+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(result);+ EsmStore store = mock(EsmStore.class);+ EventSourceMapping esm = filterEsm();++ pollerWith(store).pollAndInvoke(esm);++ verify(store, timeout(2000)).saveForAccount(eq(ACCOUNT_ID), any());+ assertEquals("s2", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));+ }+/*** A checkpoint that has aged out of the retained window names a cursor that can never succeed.* Retrying it wedges the ESM for good: later writes keep reaching the stream and none is ever