Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): drop a DynamoDB Streams batch whose OnFailure send fails (#4338)

Vũ Thế Huycee5b441d93f
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
index 1c0c8d1ba8..265f8095c5 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
@@ -72,7 +72,6 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
private final ConcurrentHashMap<String, Integer> retryCounts = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, Long> retryNotBefore = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, Integer> bisectLimits = new ConcurrentHashMap<>();
- private final ConcurrentHashMap<String, PendingFailure> pendingFailures = new ConcurrentHashMap<>();
final Set<String> stopped = ConcurrentHashMap.newKeySet();
private volatile boolean resetting;
private final ExecutorService pollExecutor = Executors.newCachedThreadPool(r -> {
@@ -196,7 +195,6 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
retryCounts.clear();
retryNotBefore.clear();
bisectLimits.clear();
- pendingFailures.clear();
}
public synchronized void startPolling(EventSourceMapping esm) {
@@ -234,7 +232,6 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
retryCounts.keySet().removeIf(k -> k.startsWith(uuid + ":"));
retryNotBefore.keySet().removeIf(k -> k.startsWith(uuid + ":"));
bisectLimits.keySet().removeIf(k -> k.startsWith(uuid + ":"));
- pendingFailures.keySet().removeIf(k -> k.startsWith(uuid + ":"));
}
@@ -248,14 +245,6 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
String shardId = DynamoDbStreamService.SHARD_ID;
String lastSeq = esm.getShardSequenceNumbers().get(shardId);
String persistedKey = batchStateKey(esm.getUuid(), shardId, lastSeq);
- PendingFailure pending = pendingFailures.get(persistedKey);
- if (pending != null) {
- long now = clock.getAsLong();
- if (now >= retryNotBefore.getOrDefault(persistedKey, 0L)) {
- dispose(esm, shardId, persistedKey, pending, now);
- }
- return;
- }
LambdaFunction fn = targetResolver.resolveMappingTarget(esm).orElse(null);
if (fn == null) {
@@ -387,17 +376,13 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
if (hasExceededMaximumRecordAge(esm, failedRecords, now)) {
LOG.warnv("DynamoDB Streams ESM {0}: maximum record age exceeded for batch ending at {1}",
esm.getUuid(), newestFetchedSeq);
- if (!dispose(esm, shardId, nextBatchKey, new PendingFailure(failedRecords, invokeResult,
- currentRetries, "MaximumRecordAgeExceeded", newestFetchedSeq), now)) {
- advanceCheckpoint(esm, shardId, nextCheckpoint);
- }
+ dispose(esm, shardId, nextBatchKey, new PendingFailure(failedRecords, invokeResult,
+ currentRetries, "MaximumRecordAgeExceeded", newestFetchedSeq));
} else if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {
LOG.warnv("DynamoDB Streams ESM {0}: maximum retry attempts ({1}) exhausted for batch ending at {2}",
esm.getUuid(), maxRetries, newestFetchedSeq);
- if (!dispose(esm, shardId, nextBatchKey, new PendingFailure(failedRecords, invokeResult,
- currentRetries, "RetryAttemptsExhausted", newestFetchedSeq), now)) {
- advanceCheckpoint(esm, shardId, nextCheckpoint);
- }
+ dispose(esm, shardId, nextBatchKey, new PendingFailure(failedRecords, invokeResult,
+ currentRetries, "RetryAttemptsExhausted", newestFetchedSeq));
} else {
retryNotBefore.put(nextBatchKey, now + retryBackoffMs(currentRetries));
LOG.warnv("DynamoDB Streams ESM {0}: Lambda returned error [batchItemFailures], retry {1}, records will be retried",
@@ -412,7 +397,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
esm.getUuid(), newestFetchedSeq);
clearBatchState(batchKey);
dispose(esm, shardId, persistedKey, new PendingFailure(matched, invokeResult,
- currentRetries, "MaximumRecordAgeExceeded", newestFetchedSeq), now);
+ currentRetries, "MaximumRecordAgeExceeded", newestFetchedSeq));
} else if (invokeResult.getFunctionError() != null
&& Boolean.TRUE.equals(esm.getBisectBatchOnFunctionError()) && matched.size() > 1) {
// ponytail: the halved limit is keyed by checkpoint, so once a good half advances the
@@ -428,7 +413,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
esm.getUuid(), maxRetries, newestFetchedSeq);
clearBatchState(batchKey);
dispose(esm, shardId, persistedKey, new PendingFailure(matched, invokeResult,
- currentRetries, "RetryAttemptsExhausted", newestFetchedSeq), now);
+ currentRetries, "RetryAttemptsExhausted", newestFetchedSeq));
} else {
retryCounts.put(batchKey, currentRetries);
retryNotBefore.put(batchKey, now + retryBackoffMs(currentRetries));
@@ -452,35 +437,22 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
private record PendingFailure(List<DynamoDbStreamRecord> records, InvokeResult invokeResult, int invokeCount,
String condition, String advanceTo) {}
- /**
- * Sends a discarded batch to the OnFailure destination and checkpoints past it. A refused send
- * parks the batch under {@code key}, the checkpoint actually persisted, and retries only the send
- * after a backoff. Returns false only when the batch was parked; a stopped mapping sends and
- * parks nothing.
- */
- private boolean dispose(EventSourceMapping esm, String shardId, String key, PendingFailure failure, long now) {
+ /** Sends a discarded batch to its OnFailure destination and checkpoints past it. */
+ private void dispose(EventSourceMapping esm, String shardId, String key, PendingFailure failure) {
if (isStopped(esm)) {
clearBatchState(key);
- return true;
- }
- if (sendToOnFailureDestination(esm, shardId, failure.records(), failure.invokeResult(),
- failure.invokeCount(), failure.condition())) {
- clearBatchState(key);
- advanceCheckpoint(esm, shardId, failure.advanceTo());
- return true;
- }
- if (pendingFailures.put(key, failure) == null) {
- retryCounts.remove(key);
+ return;
}
- retryNotBefore.put(key, now + retryBackoffMs(retryCounts.merge(key, 1, Integer::sum)));
- return false;
+ sendToOnFailureDestination(esm, shardId, failure.records(), failure.invokeResult(),
+ failure.invokeCount(), failure.condition());
+ clearBatchState(key);
+ advanceCheckpoint(esm, shardId, failure.advanceTo());
}
private void clearBatchState(String key) {
retryCounts.remove(key);
retryNotBefore.remove(key);
bisectLimits.remove(key);
- pendingFailures.remove(key);
}
private int bisectLimit(EventSourceMapping esm, String key) {
@@ -589,18 +561,17 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
return Math.min(pollIntervalMs * (1L << doublings), MAX_RETRY_BACKOFF_MS);
}
- /** Returns false only when the configured destination refused the batch. */
- private boolean sendToOnFailureDestination(EventSourceMapping esm, String shardId,
- List<DynamoDbStreamRecord> records,
- InvokeResult invokeResult,
- int invokeCount,
- String condition) {
+ private void sendToOnFailureDestination(EventSourceMapping esm, String shardId,
+ List<DynamoDbStreamRecord> records,
+ InvokeResult invokeResult,
+ int invokeCount,
+ String condition) {
if (esm.getDestinationConfig() == null || esm.getDestinationConfig().getOnFailure() == null) {
- return true;
+ return;
}
String destinationArn = esm.getDestinationConfig().getOnFailure().getDestination();
if (destinationArn == null || destinationArn.isBlank()) {
- return true;
+ return;
}
try {
@@ -629,15 +600,13 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to S3 bucket {1} with key {2}",
esm.getUuid(), arn.resource(), key);
} else {
- LOG.warnv("DynamoDB Streams ESM {0}: unsupported OnFailure destination ARN {1}",
+ LOG.warnv("DynamoDB Streams ESM {0}: unsupported OnFailure destination ARN {1}; discarding records",
esm.getUuid(), destinationArn);
}
} catch (Exception e) {
- LOG.errorv("DynamoDB Streams ESM {0}: OnFailure destination {1} refused the batch, retrying the send: {2}",
+ LOG.errorv("DynamoDB Streams ESM {0}: failed to send to OnFailure destination {1}; discarding records: {2}",
esm.getUuid(), destinationArn, e.getMessage());
- return false;
}
- return true;
}
private String buildS3OnFailurePayload(EventSourceMapping esm, String shardId,
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
index 3d1c4f11ac..82a0ce0055 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
@@ -1432,7 +1432,16 @@ class DynamoDbStreamsEventSourcePollerTest {
@ValueSource(strings = STALE_CHECKPOINT)
void bisectIsolatesThePoisonRecordAndDeliversTheRest(String startingCheckpoint) throws Exception {
stubStream("s1", "s2", "s3", "s4");
- List<List<String>> invocations = failInvocationsContaining("s3");
+ List<String> successfullyProcessedSequences = new CopyOnWriteArrayList<>();
+ List<List<String>> invocations = recordInvocations(sequences -> {
+ InvokeResult result = new InvokeResult();
+ if (sequences.contains("s3")) {
+ result.setFunctionError("Unhandled");
+ } else {
+ successfullyProcessedSequences.addAll(sequences);
+ }
+ return result;
+ });
EventSourceMapping esm = esmWithDlq(0);
esm.setBisectBatchOnFunctionError(true);
if (startingCheckpoint != null) {
@@ -1452,6 +1461,7 @@ class DynamoDbStreamsEventSourcePollerTest {
assertEquals("s3", batchInfo(body.getValue()).path("endSequenceNumber").asText());
assertEquals(1, batchInfo(body.getValue()).path("batchSize").asInt());
assertEquals("s4", checkpoint(esm));
+ assertEquals(List.of("s1", "s2", "s4"), successfullyProcessedSequences);
}
@Test
@@ -1586,7 +1596,7 @@ class DynamoDbStreamsEventSourcePollerTest {
}
@Test
- void refusedOnFailureDestinationKeepsTheBatchAndRetriesTheSend() throws Exception {
+ void refusedOnFailureDestinationDiscardsTheBatchAndAdvancesCheckpoint() throws Exception {
stubStream("s1");
List<List<String>> invocations = failInvocationsContaining("s1");
List<String> delivered = refuseSqsSends(1);
@@ -1594,40 +1604,56 @@ class DynamoDbStreamsEventSourcePollerTest {
DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
pollOnce(p, esm);
- assertNull(checkpoint(esm), "a refused send must not checkpoint the discarded batch");
- pollOnce(p, esm);
+ assertEquals("s1", checkpoint(esm));
+ assertTrue(delivered.isEmpty());
verify(sqsService, times(1)).sendMessage(anyString(), anyString(), anyInt(), anyString());
-
- advancePastRetry(p);
pollOnce(p, esm);
-
- assertEquals(List.of(List.of("s1")), invocations, "the parked batch is re-sent, not re-invoked");
- assertEquals(1, delivered.size());
- assertEquals("s1", batchInfo(delivered.get(0)).path("startSequenceNumber").asText());
- assertEquals("s1", batchInfo(delivered.get(0)).path("endSequenceNumber").asText());
+ assertEquals(1, invocations.size());
+ verify(sqsService, times(1)).sendMessage(anyString(), anyString(), anyInt(), anyString());
assertEquals("s1", checkpoint(esm));
}
@Test
- void parkedFailureIsResentAfterTheFunctionStopsResolving() throws Exception {
- stubStream("s1");
- failInvocationsContaining("s1");
+ void failedOnFailureDeliveryAllowsNextRecordsToBeProcessed() throws Exception {
+ List<DynamoDbStreamRecord> stream = stubStream("s1");
+ List<List<String>> invocations = failInvocationsContaining("s1");
List<String> delivered = refuseSqsSends(1);
EventSourceMapping esm = esmWithDlq(0);
DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
pollOnce(p, esm);
- assertNull(checkpoint(esm));
- when(functionStore.getForAccount(ACCOUNT_ID, "us-east-1", "fn")).thenReturn(Optional.empty());
- advancePastRetry(p);
+ assertEquals("s1", checkpoint(esm));
+ stream.add(ddbRecord("s2", "INSERT", "{}"));
+ pollOnce(p, esm);
+
+ assertEquals(List.of(List.of("s1"), List.of("s2")), invocations);
+ assertTrue(delivered.isEmpty());
+ verify(sqsService, times(1)).sendMessage(anyString(), anyString(), anyInt(), anyString());
+ assertEquals("s2", checkpoint(esm));
+ }
+
+ @Test
+ void unsupportedOnFailureDestinationDiscardsBatchAndAdvancesCheckpoint() throws Exception {
+ stubStream("s1");
+ failInvocationsContaining("s1");
+ EventSourceMapping esm = filterEsm();
+ esm.setMaximumRetryAttempts(0);
+ EventSourceMapping.OnFailure onFailure = new EventSourceMapping.OnFailure();
+ onFailure.setDestination("arn:aws:lambda:us-east-1:000000000000:function:unsupported");
+ EventSourceMapping.DestinationConfig destinationConfig = new EventSourceMapping.DestinationConfig();
+ destinationConfig.setOnFailure(onFailure);
+ esm.setDestinationConfig(destinationConfig);
+
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
pollOnce(p, esm);
- assertEquals(1, delivered.size(), "the resend does not need the function");
assertEquals("s1", checkpoint(esm));
+ verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString());
+ verify(snsService, never()).publish(anyString(), any(), anyString(), anyString(), anyString());
}
@Test
- void refusedOnFailureDestinationAfterPartialSuccessKeepsTheAcknowledgedPrefix() throws Exception {
+ void refusedOnFailureDestinationDiscardsFailedSuffixAfterPartialSuccess() throws Exception {
stubStream("s1", "s2", "s3");
List<List<String>> invocations = recordInvocations(seqs -> partialFailure("s2"));
List<String> delivered = refuseSqsSends(1);
@@ -1636,15 +1662,14 @@ class DynamoDbStreamsEventSourcePollerTest {
DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
pollOnce(p, esm);
- assertEquals("s1", checkpoint(esm));
+ assertEquals("s3", checkpoint(esm));
advancePastRetry(p);
pollOnce(p, esm);
assertEquals(1, invocations.size());
- assertEquals(1, delivered.size());
- assertEquals("s2", batchInfo(delivered.get(0)).path("startSequenceNumber").asText());
- assertEquals("s3", batchInfo(delivered.get(0)).path("endSequenceNumber").asText());
+ assertTrue(delivered.isEmpty());
+ verify(sqsService, times(1)).sendMessage(anyString(), anyString(), anyInt(), anyString());
assertEquals("s3", checkpoint(esm));
}
@@ -1924,26 +1949,6 @@ class DynamoDbStreamsEventSourcePollerTest {
verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString());
}
- @Test
- void parkedFailureIsNotResentOnceTheMappingIsStopped() throws Exception {
- stubStream("s1");
- List<List<String>> invocations = failInvocationsContaining("s1");
- refuseSqsSends(1);
- EventSourceMapping esm = esmWithDlq(0);
- DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
- pollOnce(p, esm);
- assertNull(checkpoint(esm));
- // A worker that loaded the parked entry before stopPolling cleared it sees only the tombstone.
- p.stopped.add(esm.getUuid());
-
- advancePastRetry(p);
- pollOnce(p, esm);
-
- verify(sqsService, times(1)).sendMessage(anyString(), anyString(), anyInt(), anyString());
- assertNull(checkpoint(esm));
- assertEquals(List.of(List.of("s1")), invocations);
- }
-
@Test
void checkpointIsNotSavedWhileAResetIsInProgress() throws Exception {
stubStream("s1");