Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): honor DynamoDB stream partial batch failures (#3466)

XuJianc6316d10cffa
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
index 21910db8a5..2d048d76c9 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
@@ -21,7 +21,11 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -223,7 +227,11 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
}
if (invokeResult.getFunctionError() == null) {
- advanceCheckpoint(esm, shardId, newestFetchedSeq);
+ String checkpoint = successfulInvocationCheckpoint(
+ esm, invokeResult, lastSeq, records, matched);
+ if (checkpoint != null && !checkpoint.equals(lastSeq)) {
+ advanceCheckpoint(esm, shardId, checkpoint);
+ }
} else {
LOG.warnv("DynamoDB Streams ESM {0}: Lambda returned error [{1}], records will be retried",
esm.getUuid(), invokeResult.getFunctionError());
@@ -236,6 +244,75 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
});
}
+ /**
+ * Returns the last record that can be consumed after a successful invocation. Floci stores the
+ * last consumed sequence and resumes with {@code AFTER_SEQUENCE_NUMBER}, so a partial failure
+ * checkpoints the record immediately before AWS's lowest reported failed sequence.
+ */
+ private String successfulInvocationCheckpoint(EventSourceMapping esm, InvokeResult invokeResult,
+ String previousCheckpoint,
+ List<DynamoDbStreamRecord> fetched,
+ List<DynamoDbStreamRecord> delivered) {
+ String newestFetchedSeq = fetched.get(fetched.size() - 1).getSequenceNumber();
+ byte[] payload = invokeResult.getPayload();
+ if (!esm.isReportBatchItemFailures() || payload == null || payload.length == 0) {
+ return newestFetchedSeq;
+ }
+
+ try {
+ JsonNode response = objectMapper.readTree(payload);
+ JsonNode failures = response.get("batchItemFailures");
+ if (failures == null || failures.isNull()) {
+ return newestFetchedSeq;
+ }
+ if (!failures.isArray()) {
+ return retryWholeBatch(esm, previousCheckpoint, "batchItemFailures is not an array");
+ }
+
+ Map<String, Integer> fetchedIndexes = new HashMap<>();
+ for (int i = 0; i < fetched.size(); i++) {
+ fetchedIndexes.put(fetched.get(i).getSequenceNumber(), i);
+ }
+ Set<String> deliveredSequences = new HashSet<>();
+ for (DynamoDbStreamRecord record : delivered) {
+ deliveredSequences.add(record.getSequenceNumber());
+ }
+
+ int lowestFailedIndex = fetched.size();
+ for (JsonNode item : failures) {
+ JsonNode identifier = item.get("itemIdentifier");
+ if (identifier == null || identifier.isNull() || identifier.asText().isEmpty()) {
+ return retryWholeBatch(esm, previousCheckpoint,
+ "entry has a missing, null or empty itemIdentifier");
+ }
+ String sequenceNumber = identifier.asText();
+ Integer index = fetchedIndexes.get(sequenceNumber);
+ if (index == null || !deliveredSequences.contains(sequenceNumber)) {
+ return retryWholeBatch(esm, previousCheckpoint,
+ "itemIdentifier " + sequenceNumber + " is not in the delivered batch");
+ }
+ lowestFailedIndex = Math.min(lowestFailedIndex, index);
+ }
+
+ if (lowestFailedIndex == fetched.size()) {
+ return newestFetchedSeq;
+ }
+ return lowestFailedIndex == 0
+ ? previousCheckpoint
+ : fetched.get(lowestFailedIndex - 1).getSequenceNumber();
+ } catch (Exception e) {
+ return retryWholeBatch(esm, previousCheckpoint,
+ "response is not valid JSON: " + e.getMessage());
+ }
+ }
+
+ private String retryWholeBatch(EventSourceMapping esm, String previousCheckpoint, String reason) {
+ LOG.warnv("DynamoDB Streams ESM {0}: malformed batchItemFailures response ({1}), "
+ + "retrying the whole batch",
+ esm.getUuid(), reason);
+ return previousCheckpoint;
+ }
+
private String buildDynamoDbEvent(List<DynamoDbStreamRecord> records, EventSourceMapping esm) {
try {
ObjectNode root = objectMapper.createObjectNode();
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
index 289d3d3ba2..44ab2ecf9d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
@@ -332,6 +332,91 @@ class DynamoDbStreamsEventSourcePollerTest {
assertEquals("s2", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));
}
+ @Test
+ void partialBatchFailureCheckpointsBeforeLowestFailedRecord() {
+ stubTrimHorizon(List.of(
+ ddbRecord("s1", "INSERT", "{}"),
+ ddbRecord("s2", "INSERT", "{}"),
+ ddbRecord("s3", "INSERT", "{}")));
+ InvokeResult result = new InvokeResult();
+ result.setPayload("{\"batchItemFailures\":[{\"itemIdentifier\":\"s3\"},{\"itemIdentifier\":\"s2\"}]}".getBytes());
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(result);
+ EsmStore store = mock(EsmStore.class);
+ EventSourceMapping esm = filterEsm();
+ esm.setFunctionResponseTypes(List.of("ReportBatchItemFailures"));
+
+ pollerWith(store).pollAndInvoke(esm);
+
+ verify(store, timeout(2000)).saveForAccount(eq(ACCOUNT_ID), any());
+ assertEquals("s1", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));
+ }
+
+ @Test
+ void partialBatchFailureAtFirstRecordKeepsPriorCheckpoint() {
+ stubTrimHorizon(List.of(
+ ddbRecord("s1", "INSERT", "{}"),
+ ddbRecord("s2", "INSERT", "{}")));
+ InvokeResult result = new InvokeResult();
+ result.setPayload("{\"batchItemFailures\":[{\"itemIdentifier\":\"s1\"}]}".getBytes());
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(result);
+ EsmStore store = mock(EsmStore.class);
+ EventSourceMapping esm = filterEsm();
+ esm.setFunctionResponseTypes(List.of("ReportBatchItemFailures"));
+ esm.getShardSequenceNumbers().put(DynamoDbStreamService.SHARD_ID, "s0");
+ DynamoDbStreamsEventSourcePoller p = pollerWith(store);
+
+ p.pollAndInvoke(esm);
+ verify(executorService, timeout(2000))
+ .invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));
+ awaitPollCompletedViaSecondFetch(p, esm);
+
+ verify(store, never()).saveForAccount(anyString(), any());
+ assertEquals("s0", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));
+ }
+
+ @Test
+ void malformedPartialBatchResponseRetriesWholeBatch() {
+ stubTrimHorizon(List.of(
+ ddbRecord("s1", "INSERT", "{}"),
+ ddbRecord("s2", "INSERT", "{}")));
+ InvokeResult result = new InvokeResult();
+ result.setPayload("{\"batchItemFailures\":[{\"itemIdentifier\":\"unknown\"}]}".getBytes());
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(result);
+ EsmStore store = mock(EsmStore.class);
+ EventSourceMapping esm = filterEsm();
+ esm.setFunctionResponseTypes(List.of("ReportBatchItemFailures"));
+ DynamoDbStreamsEventSourcePoller p = pollerWith(store);
+
+ p.pollAndInvoke(esm);
+ verify(executorService, timeout(2000))
+ .invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));
+ awaitPollCompletedViaSecondFetch(p, esm);
+
+ verify(store, never()).saveForAccount(anyString(), any());
+ assertNull(esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));
+ }
+
+ @Test
+ void partialBatchResponseIsIgnoredWhenNotConfigured() {
+ stubTrimHorizon(List.of(
+ ddbRecord("s1", "INSERT", "{}"),
+ ddbRecord("s2", "INSERT", "{}")));
+ InvokeResult result = new InvokeResult();
+ result.setPayload("{\"batchItemFailures\":[{\"itemIdentifier\":\"s1\"}]}".getBytes());
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(result);
+ EsmStore store = mock(EsmStore.class);
+ EventSourceMapping esm = filterEsm();
+
+ pollerWith(store).pollAndInvoke(esm);
+
+ verify(store, timeout(2000)).saveForAccount(eq(ACCOUNT_ID), any());
+ assertEquals("s2", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));
+ }
+
/**
* A checkpoint that has aged out of the retained window names a cursor that can never succeed.
* Retrying it wedges the ESM for good: later writes keep reaching the stream and none is ever