Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): drop a DynamoDB Streams batch whose OnFailure send fails (#4338)
Vũ Thế Huycee5b441d93f
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.javaindex 1c0c8d1ba8..265f8095c5 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java@@ -72,7 +72,6 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {private final ConcurrentHashMap<String, Integer> retryCounts = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, Long> retryNotBefore = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, Integer> bisectLimits = new ConcurrentHashMap<>();- private final ConcurrentHashMap<String, PendingFailure> pendingFailures = new ConcurrentHashMap<>();final Set<String> stopped = ConcurrentHashMap.newKeySet();private volatile boolean resetting;private final ExecutorService pollExecutor = Executors.newCachedThreadPool(r -> {@@ -196,7 +195,6 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {retryCounts.clear();retryNotBefore.clear();bisectLimits.clear();- pendingFailures.clear();}public synchronized void startPolling(EventSourceMapping esm) {@@ -234,7 +232,6 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {retryCounts.keySet().removeIf(k -> k.startsWith(uuid + ":"));retryNotBefore.keySet().removeIf(k -> k.startsWith(uuid + ":"));bisectLimits.keySet().removeIf(k -> k.startsWith(uuid + ":"));- pendingFailures.keySet().removeIf(k -> k.startsWith(uuid + ":"));}@@ -248,14 +245,6 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {String shardId = DynamoDbStreamService.SHARD_ID;String lastSeq = esm.getShardSequenceNumbers().get(shardId);String persistedKey = batchStateKey(esm.getUuid(), shardId, lastSeq);- PendingFailure pending = pendingFailures.get(persistedKey);- if (pending != null) {- long now = clock.getAsLong();- if (now >= retryNotBefore.getOrDefault(persistedKey, 0L)) {- dispose(esm, shardId, persistedKey, pending, now);- }- return;- }LambdaFunction fn = targetResolver.resolveMappingTarget(esm).orElse(null);if (fn == null) {@@ -387,17 +376,13 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {if (hasExceededMaximumRecordAge(esm, failedRecords, now)) {LOG.warnv("DynamoDB Streams ESM {0}: maximum record age exceeded for batch ending at {1}",esm.getUuid(), newestFetchedSeq);- if (!dispose(esm, shardId, nextBatchKey, new PendingFailure(failedRecords, invokeResult,- currentRetries, "MaximumRecordAgeExceeded", newestFetchedSeq), now)) {- advanceCheckpoint(esm, shardId, nextCheckpoint);- }+ dispose(esm, shardId, nextBatchKey, new PendingFailure(failedRecords, invokeResult,+ currentRetries, "MaximumRecordAgeExceeded", newestFetchedSeq));} else if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {LOG.warnv("DynamoDB Streams ESM {0}: maximum retry attempts ({1}) exhausted for batch ending at {2}",esm.getUuid(), maxRetries, newestFetchedSeq);- if (!dispose(esm, shardId, nextBatchKey, new PendingFailure(failedRecords, invokeResult,- currentRetries, "RetryAttemptsExhausted", newestFetchedSeq), now)) {- advanceCheckpoint(esm, shardId, nextCheckpoint);- }+ dispose(esm, shardId, nextBatchKey, new PendingFailure(failedRecords, invokeResult,+ currentRetries, "RetryAttemptsExhausted", newestFetchedSeq));} else {retryNotBefore.put(nextBatchKey, now + retryBackoffMs(currentRetries));LOG.warnv("DynamoDB Streams ESM {0}: Lambda returned error [batchItemFailures], retry {1}, records will be retried",@@ -412,7 +397,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {esm.getUuid(), newestFetchedSeq);clearBatchState(batchKey);dispose(esm, shardId, persistedKey, new PendingFailure(matched, invokeResult,- currentRetries, "MaximumRecordAgeExceeded", newestFetchedSeq), now);+ currentRetries, "MaximumRecordAgeExceeded", newestFetchedSeq));} else if (invokeResult.getFunctionError() != null&& Boolean.TRUE.equals(esm.getBisectBatchOnFunctionError()) && matched.size() > 1) {// ponytail: the halved limit is keyed by checkpoint, so once a good half advances the@@ -428,7 +413,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {esm.getUuid(), maxRetries, newestFetchedSeq);clearBatchState(batchKey);dispose(esm, shardId, persistedKey, new PendingFailure(matched, invokeResult,- currentRetries, "RetryAttemptsExhausted", newestFetchedSeq), now);+ currentRetries, "RetryAttemptsExhausted", newestFetchedSeq));} else {retryCounts.put(batchKey, currentRetries);retryNotBefore.put(batchKey, now + retryBackoffMs(currentRetries));@@ -452,35 +437,22 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {private record PendingFailure(List<DynamoDbStreamRecord> records, InvokeResult invokeResult, int invokeCount,String condition, String advanceTo) {}- /**- * Sends a discarded batch to the OnFailure destination and checkpoints past it. A refused send- * parks the batch under {@code key}, the checkpoint actually persisted, and retries only the send- * after a backoff. Returns false only when the batch was parked; a stopped mapping sends and- * parks nothing.- */- private boolean dispose(EventSourceMapping esm, String shardId, String key, PendingFailure failure, long now) {+ /** Sends a discarded batch to its OnFailure destination and checkpoints past it. */+ private void dispose(EventSourceMapping esm, String shardId, String key, PendingFailure failure) {if (isStopped(esm)) {clearBatchState(key);- return true;- }- if (sendToOnFailureDestination(esm, shardId, failure.records(), failure.invokeResult(),- failure.invokeCount(), failure.condition())) {- clearBatchState(key);- advanceCheckpoint(esm, shardId, failure.advanceTo());- return true;- }- if (pendingFailures.put(key, failure) == null) {- retryCounts.remove(key);+ return;}- retryNotBefore.put(key, now + retryBackoffMs(retryCounts.merge(key, 1, Integer::sum)));- return false;+ sendToOnFailureDestination(esm, shardId, failure.records(), failure.invokeResult(),+ failure.invokeCount(), failure.condition());+ clearBatchState(key);+ advanceCheckpoint(esm, shardId, failure.advanceTo());}private void clearBatchState(String key) {retryCounts.remove(key);retryNotBefore.remove(key);bisectLimits.remove(key);- pendingFailures.remove(key);}private int bisectLimit(EventSourceMapping esm, String key) {@@ -589,18 +561,17 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {return Math.min(pollIntervalMs * (1L << doublings), MAX_RETRY_BACKOFF_MS);}- /** Returns false only when the configured destination refused the batch. */- private boolean sendToOnFailureDestination(EventSourceMapping esm, String shardId,- List<DynamoDbStreamRecord> records,- InvokeResult invokeResult,- int invokeCount,- String condition) {+ private void sendToOnFailureDestination(EventSourceMapping esm, String shardId,+ List<DynamoDbStreamRecord> records,+ InvokeResult invokeResult,+ int invokeCount,+ String condition) {if (esm.getDestinationConfig() == null || esm.getDestinationConfig().getOnFailure() == null) {- return true;+ return;}String destinationArn = esm.getDestinationConfig().getOnFailure().getDestination();if (destinationArn == null || destinationArn.isBlank()) {- return true;+ return;}try {@@ -629,15 +600,13 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to S3 bucket {1} with key {2}",esm.getUuid(), arn.resource(), key);} else {- LOG.warnv("DynamoDB Streams ESM {0}: unsupported OnFailure destination ARN {1}",+ LOG.warnv("DynamoDB Streams ESM {0}: unsupported OnFailure destination ARN {1}; discarding records",esm.getUuid(), destinationArn);}} catch (Exception e) {- LOG.errorv("DynamoDB Streams ESM {0}: OnFailure destination {1} refused the batch, retrying the send: {2}",+ LOG.errorv("DynamoDB Streams ESM {0}: failed to send to OnFailure destination {1}; discarding records: {2}",esm.getUuid(), destinationArn, e.getMessage());- return false;}- return true;}private String buildS3OnFailurePayload(EventSourceMapping esm, String shardId,diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.javaindex 3d1c4f11ac..82a0ce0055 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java@@ -1432,7 +1432,16 @@ class DynamoDbStreamsEventSourcePollerTest {@ValueSource(strings = STALE_CHECKPOINT)void bisectIsolatesThePoisonRecordAndDeliversTheRest(String startingCheckpoint) throws Exception {stubStream("s1", "s2", "s3", "s4");- List<List<String>> invocations = failInvocationsContaining("s3");+ List<String> successfullyProcessedSequences = new CopyOnWriteArrayList<>();+ List<List<String>> invocations = recordInvocations(sequences -> {+ InvokeResult result = new InvokeResult();+ if (sequences.contains("s3")) {+ result.setFunctionError("Unhandled");+ } else {+ successfullyProcessedSequences.addAll(sequences);+ }+ return result;+ });EventSourceMapping esm = esmWithDlq(0);esm.setBisectBatchOnFunctionError(true);if (startingCheckpoint != null) {@@ -1452,6 +1461,7 @@ class DynamoDbStreamsEventSourcePollerTest {assertEquals("s3", batchInfo(body.getValue()).path("endSequenceNumber").asText());assertEquals(1, batchInfo(body.getValue()).path("batchSize").asInt());assertEquals("s4", checkpoint(esm));+ assertEquals(List.of("s1", "s2", "s4"), successfullyProcessedSequences);}@Test@@ -1586,7 +1596,7 @@ class DynamoDbStreamsEventSourcePollerTest {}@Test- void refusedOnFailureDestinationKeepsTheBatchAndRetriesTheSend() throws Exception {+ void refusedOnFailureDestinationDiscardsTheBatchAndAdvancesCheckpoint() throws Exception {stubStream("s1");List<List<String>> invocations = failInvocationsContaining("s1");List<String> delivered = refuseSqsSends(1);@@ -1594,40 +1604,56 @@ class DynamoDbStreamsEventSourcePollerTest {DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));pollOnce(p, esm);- assertNull(checkpoint(esm), "a refused send must not checkpoint the discarded batch");- pollOnce(p, esm);+ assertEquals("s1", checkpoint(esm));+ assertTrue(delivered.isEmpty());verify(sqsService, times(1)).sendMessage(anyString(), anyString(), anyInt(), anyString());-- advancePastRetry(p);pollOnce(p, esm);-- assertEquals(List.of(List.of("s1")), invocations, "the parked batch is re-sent, not re-invoked");- assertEquals(1, delivered.size());- assertEquals("s1", batchInfo(delivered.get(0)).path("startSequenceNumber").asText());- assertEquals("s1", batchInfo(delivered.get(0)).path("endSequenceNumber").asText());+ assertEquals(1, invocations.size());+ verify(sqsService, times(1)).sendMessage(anyString(), anyString(), anyInt(), anyString());assertEquals("s1", checkpoint(esm));}@Test- void parkedFailureIsResentAfterTheFunctionStopsResolving() throws Exception {- stubStream("s1");- failInvocationsContaining("s1");+ void failedOnFailureDeliveryAllowsNextRecordsToBeProcessed() throws Exception {+ List<DynamoDbStreamRecord> stream = stubStream("s1");+ List<List<String>> invocations = failInvocationsContaining("s1");List<String> delivered = refuseSqsSends(1);EventSourceMapping esm = esmWithDlq(0);DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));pollOnce(p, esm);- assertNull(checkpoint(esm));- when(functionStore.getForAccount(ACCOUNT_ID, "us-east-1", "fn")).thenReturn(Optional.empty());- advancePastRetry(p);+ assertEquals("s1", checkpoint(esm));+ stream.add(ddbRecord("s2", "INSERT", "{}"));+ pollOnce(p, esm);++ assertEquals(List.of(List.of("s1"), List.of("s2")), invocations);+ assertTrue(delivered.isEmpty());+ verify(sqsService, times(1)).sendMessage(anyString(), anyString(), anyInt(), anyString());+ assertEquals("s2", checkpoint(esm));+ }++ @Test+ void unsupportedOnFailureDestinationDiscardsBatchAndAdvancesCheckpoint() throws Exception {+ stubStream("s1");+ failInvocationsContaining("s1");+ EventSourceMapping esm = filterEsm();+ esm.setMaximumRetryAttempts(0);+ EventSourceMapping.OnFailure onFailure = new EventSourceMapping.OnFailure();+ onFailure.setDestination("arn:aws:lambda:us-east-1:000000000000:function:unsupported");+ EventSourceMapping.DestinationConfig destinationConfig = new EventSourceMapping.DestinationConfig();+ destinationConfig.setOnFailure(onFailure);+ esm.setDestinationConfig(destinationConfig);++ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));pollOnce(p, esm);- assertEquals(1, delivered.size(), "the resend does not need the function");assertEquals("s1", checkpoint(esm));+ verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString());+ verify(snsService, never()).publish(anyString(), any(), anyString(), anyString(), anyString());}@Test- void refusedOnFailureDestinationAfterPartialSuccessKeepsTheAcknowledgedPrefix() throws Exception {+ void refusedOnFailureDestinationDiscardsFailedSuffixAfterPartialSuccess() throws Exception {stubStream("s1", "s2", "s3");List<List<String>> invocations = recordInvocations(seqs -> partialFailure("s2"));List<String> delivered = refuseSqsSends(1);@@ -1636,15 +1662,14 @@ class DynamoDbStreamsEventSourcePollerTest {DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));pollOnce(p, esm);- assertEquals("s1", checkpoint(esm));+ assertEquals("s3", checkpoint(esm));advancePastRetry(p);pollOnce(p, esm);assertEquals(1, invocations.size());- assertEquals(1, delivered.size());- assertEquals("s2", batchInfo(delivered.get(0)).path("startSequenceNumber").asText());- assertEquals("s3", batchInfo(delivered.get(0)).path("endSequenceNumber").asText());+ assertTrue(delivered.isEmpty());+ verify(sqsService, times(1)).sendMessage(anyString(), anyString(), anyInt(), anyString());assertEquals("s3", checkpoint(esm));}@@ -1924,26 +1949,6 @@ class DynamoDbStreamsEventSourcePollerTest {verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString());}- @Test- void parkedFailureIsNotResentOnceTheMappingIsStopped() throws Exception {- stubStream("s1");- List<List<String>> invocations = failInvocationsContaining("s1");- refuseSqsSends(1);- EventSourceMapping esm = esmWithDlq(0);- DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));- pollOnce(p, esm);- assertNull(checkpoint(esm));- // A worker that loaded the parked entry before stopPolling cleared it sees only the tombstone.- p.stopped.add(esm.getUuid());-- advancePastRetry(p);- pollOnce(p, esm);-- verify(sqsService, times(1)).sendMessage(anyString(), anyString(), anyInt(), anyString());- assertNull(checkpoint(esm));- assertEquals(List.of(List.of("s1")), invocations);- }-@Testvoid checkpointIsNotSavedWhileAResetIsInProgress() throws Exception {stubStream("s1");