Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(lambda): bisect failing DynamoDB stream batches and keep refused failures (#4337)

Mario Toffia88a992757e2b
diff --git a/docs/services/lambda.md b/docs/services/lambda.md
index 8b22263bf2..83dc78da79 100644
--- a/docs/services/lambda.md
+++ b/docs/services/lambda.md
@@ -834,7 +834,7 @@ aws lambda update-function-code \
Connect Lambda to SQS, Kinesis, or DynamoDB Streams. Self-managed Apache Kafka event source mappings are accepted, validated, persisted, and returned on the wire, but Floci does not run an active Kafka consumer poller:
-For DynamoDB Streams mappings, Floci retries failed batches with exponential backoff, honors `MaximumRetryAttempts` and `MaximumRecordAgeInSeconds`, and sends discarded batches to configured SQS or SNS `DestinationConfig.OnFailure` destinations.
+For DynamoDB Streams mappings, Floci retries failed batches with exponential backoff, honors `MaximumRetryAttempts` and `MaximumRecordAgeInSeconds`, and sends discarded batches to configured SQS, SNS, or S3 `DestinationConfig.OnFailure` destinations. With `BisectBatchOnFunctionError`, a batch that fails with a function error is split in half and retried, narrowing it down to the failing record, which then follows the normal retry and record age rules. Splits do not count as retry attempts, and only function errors split: throttles do not, and a partial batch response retries from the reported record. If the OnFailure destination refuses a discarded batch, Floci keeps the checkpoint and retries the send with backoff instead of advancing past the batch.
```bash
# SQS trigger
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
index c9963764db..f0792f9f88 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
@@ -71,6 +71,8 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
final ConcurrentHashMap<String, Boolean> activePolls = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, Integer> retryCounts = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, Long> retryNotBefore = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String, Integer> bisectLimits = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String, PendingFailure> pendingFailures = new ConcurrentHashMap<>();
private final ExecutorService pollExecutor = Executors.newCachedThreadPool(r -> {
Thread t = new Thread(r, "dynamodb-streams-esm-poller");
t.setDaemon(true);
@@ -166,6 +168,8 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
activePolls.clear();
retryCounts.clear();
retryNotBefore.clear();
+ bisectLimits.clear();
+ pendingFailures.clear();
}
public void startPolling(EventSourceMapping esm) {
@@ -192,6 +196,8 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
}
retryCounts.keySet().removeIf(k -> k.startsWith(uuid + ":"));
retryNotBefore.keySet().removeIf(k -> k.startsWith(uuid + ":"));
+ bisectLimits.keySet().removeIf(k -> k.startsWith(uuid + ":"));
+ pendingFailures.keySet().removeIf(k -> k.startsWith(uuid + ":"));
}
@@ -201,6 +207,19 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
}
pollExecutor.submit(() -> {
try {
+ String streamArn = esm.getEventSourceArn();
+ String shardId = DynamoDbStreamService.SHARD_ID;
+ String lastSeq = esm.getShardSequenceNumbers().get(shardId);
+ String persistedKey = batchStateKey(esm.getUuid(), shardId, lastSeq);
+ PendingFailure pending = pendingFailures.get(persistedKey);
+ if (pending != null) {
+ long now = clock.getAsLong();
+ if (now >= retryNotBefore.getOrDefault(persistedKey, 0L)) {
+ dispose(esm, shardId, persistedKey, pending, now);
+ }
+ return;
+ }
+
LambdaFunction fn = targetResolver.resolveMappingTarget(esm).orElse(null);
if (fn == null) {
LOG.warnv("DynamoDB Streams ESM {0}: function {1} not found, skipping",
@@ -208,22 +227,20 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
return;
}
- String streamArn = esm.getEventSourceArn();
- String shardId = DynamoDbStreamService.SHARD_ID;
- String lastSeq = esm.getShardSequenceNumbers().get(shardId);
-
boolean checkpointTrimmed = false;
DynamoDbStreamService.GetRecordsResult result;
try {
String iterator = lastSeq == null
? streamService.getShardIterator(streamArn, shardId, "TRIM_HORIZON", null)
: streamService.getShardIterator(streamArn, shardId, "AFTER_SEQUENCE_NUMBER", lastSeq);
- result = streamService.getRecords(iterator, esm.getBatchSize());
+ result = streamService.getRecords(iterator,
+ Math.min(bisectLimit(esm, persistedKey), esm.getBatchSize()));
} catch (AwsException e) {
if (!TRIMMED_DATA_ACCESS_EXCEPTION.equals(e.getErrorCode())) {
throw e;
}
checkpointTrimmed = true;
+ clearBatchState(persistedKey);
// The checkpoint fell outside the retained window, so the cursor it names can
// never succeed again. Retrying it wedges the ESM permanently: every later
// write reaches the stream and none is ever delivered. Resume from the oldest
@@ -234,7 +251,8 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
+ "trim horizon; records between were dropped from the stream",
esm.getUuid(), lastSeq);
String horizon = streamService.getShardIterator(streamArn, shardId, "TRIM_HORIZON", null);
- result = streamService.getRecords(horizon, esm.getBatchSize());
+ result = streamService.getRecords(horizon, Math.min(esm.getBatchSize(),
+ bisectLimit(esm, batchStateKey(esm.getUuid(), shardId, null))));
}
List<DynamoDbStreamRecord> records = result.records();
@@ -258,13 +276,13 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
records, filterNodes, filterMatcher.applyFilterCriteria(filterNodes, filterParams));
}
+ String batchKey = batchStateKey(esm.getUuid(), shardId, checkpointTrimmed ? null : lastSeq);
if (matched.isEmpty()) {
+ clearBatchState(batchKey);
advanceCheckpoint(esm, shardId, newestFetchedSeq);
return;
}
- String checkpointSeq = (lastSeq == null || checkpointTrimmed) ? "TRIM_HORIZON" : lastSeq;
- String batchKey = esm.getUuid() + ":" + shardId + ":" + checkpointSeq;
long now = clock.getAsLong();
if (now < retryNotBefore.getOrDefault(batchKey, 0L)) {
return;
@@ -291,14 +309,13 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
: new CheckpointOutcome(null, 0);
if (outcome.lowestFailedIndex() == records.size()) {
- retryCounts.remove(batchKey);
- retryNotBefore.remove(batchKey);
+ clearBatchState(batchKey);
advanceCheckpoint(esm, shardId, outcome.checkpoint());
} else if (outcome.lowestFailedIndex() > 0) {
- retryCounts.remove(batchKey);
+ clearBatchState(batchKey);
String nextCheckpoint = outcome.checkpoint();
- String nextBatchKey = esm.getUuid() + ":" + shardId + ":" + nextCheckpoint;
+ String nextBatchKey = batchStateKey(esm.getUuid(), shardId, nextCheckpoint);
Integer maxRetries = esm.getMaximumRetryAttempts();
int currentRetries = retryCounts.merge(nextBatchKey, 1, Integer::sum);
Set<String> deliveredSeqs = new HashSet<>();
@@ -319,19 +336,17 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
if (hasExceededMaximumRecordAge(esm, failedRecords, now)) {
LOG.warnv("DynamoDB Streams ESM {0}: maximum record age exceeded for batch ending at {1}",
esm.getUuid(), newestFetchedSeq);
- sendToOnFailureDestination(esm, shardId, failedRecords, invokeResult, currentRetries,
- "MaximumRecordAgeExceeded");
- retryCounts.remove(nextBatchKey);
- retryNotBefore.remove(nextBatchKey);
- advanceCheckpoint(esm, shardId, newestFetchedSeq);
+ if (!dispose(esm, shardId, nextBatchKey, new PendingFailure(failedRecords, invokeResult,
+ currentRetries, "MaximumRecordAgeExceeded", newestFetchedSeq), now)) {
+ advanceCheckpoint(esm, shardId, nextCheckpoint);
+ }
} else if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {
LOG.warnv("DynamoDB Streams ESM {0}: maximum retry attempts ({1}) exhausted for batch ending at {2}",
esm.getUuid(), maxRetries, newestFetchedSeq);
- sendToOnFailureDestination(esm, shardId, failedRecords, invokeResult, currentRetries,
- "RetryAttemptsExhausted");
- retryCounts.remove(nextBatchKey);
- retryNotBefore.remove(nextBatchKey);
- advanceCheckpoint(esm, shardId, newestFetchedSeq);
+ if (!dispose(esm, shardId, nextBatchKey, new PendingFailure(failedRecords, invokeResult,
+ currentRetries, "RetryAttemptsExhausted", newestFetchedSeq), now)) {
+ advanceCheckpoint(esm, shardId, nextCheckpoint);
+ }
} else {
retryNotBefore.put(nextBatchKey, now + retryBackoffMs(currentRetries));
LOG.warnv("DynamoDB Streams ESM {0}: Lambda returned error [batchItemFailures], retry {1}, records will be retried",
@@ -340,24 +355,31 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
}
} else {
Integer maxRetries = esm.getMaximumRetryAttempts();
- int currentRetries = retryCounts.merge(batchKey, 1, Integer::sum);
+ int currentRetries = retryCounts.getOrDefault(batchKey, 0) + 1;
if (hasExceededMaximumRecordAge(esm, matched, now)) {
LOG.warnv("DynamoDB Streams ESM {0}: maximum record age exceeded for batch ending at {1}",
esm.getUuid(), newestFetchedSeq);
- sendToOnFailureDestination(esm, shardId, matched, invokeResult, currentRetries,
- "MaximumRecordAgeExceeded");
- retryCounts.remove(batchKey);
- retryNotBefore.remove(batchKey);
- advanceCheckpoint(esm, shardId, newestFetchedSeq);
+ clearBatchState(batchKey);
+ dispose(esm, shardId, persistedKey, new PendingFailure(matched, invokeResult,
+ currentRetries, "MaximumRecordAgeExceeded", newestFetchedSeq), now);
+ } else if (invokeResult.getFunctionError() != null
+ && Boolean.TRUE.equals(esm.getBisectBatchOnFunctionError()) && matched.size() > 1) {
+ // ponytail: the halved limit is keyed by checkpoint, so once a good half advances the
+ // rest is refetched at full batch size and split again. One window converges on the
+ // poison record in O(log n) invocations, a busy stream in O(log^2 n) worst case;
+ // carrying the right half's end sequence forward would match AWS exactly.
+ int limit = (records.size() + 1) / 2;
+ bisectLimits.put(batchKey, limit);
+ LOG.infov("DynamoDB Streams ESM {0}: function error on {1} record(s), bisecting to {2}",
+ esm.getUuid(), records.size(), limit);
} else if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {
LOG.warnv("DynamoDB Streams ESM {0}: maximum retry attempts ({1}) exhausted for batch ending at {2}",
esm.getUuid(), maxRetries, newestFetchedSeq);
- sendToOnFailureDestination(esm, shardId, matched, invokeResult, currentRetries,
- "RetryAttemptsExhausted");
- retryCounts.remove(batchKey);
- retryNotBefore.remove(batchKey);
- advanceCheckpoint(esm, shardId, newestFetchedSeq);
+ clearBatchState(batchKey);
+ dispose(esm, shardId, persistedKey, new PendingFailure(matched, invokeResult,
+ currentRetries, "RetryAttemptsExhausted", newestFetchedSeq), now);
} else {
+ retryCounts.put(batchKey, currentRetries);
retryNotBefore.put(batchKey, now + retryBackoffMs(currentRetries));
String error = invokeResult.getFunctionError() != null
? invokeResult.getFunctionError()
@@ -376,6 +398,45 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
private record CheckpointOutcome(String checkpoint, int lowestFailedIndex) {}
+ private record PendingFailure(List<DynamoDbStreamRecord> records, InvokeResult invokeResult, int invokeCount,
+ String condition, String advanceTo) {}
+
+ /**
+ * Sends a discarded batch to the OnFailure destination and checkpoints past it. A refused send
+ * parks the batch under {@code key}, the checkpoint actually persisted, and retries only the send
+ * after a backoff. Returns false when the batch was parked.
+ */
+ private boolean dispose(EventSourceMapping esm, String shardId, String key, PendingFailure failure, long now) {
+ if (sendToOnFailureDestination(esm, shardId, failure.records(), failure.invokeResult(),
+ failure.invokeCount(), failure.condition())) {
+ clearBatchState(key);
+ advanceCheckpoint(esm, shardId, failure.advanceTo());
+ return true;
+ }
+ if (pendingFailures.put(key, failure) == null) {
+ retryCounts.remove(key);
+ }
+ retryNotBefore.put(key, now + retryBackoffMs(retryCounts.merge(key, 1, Integer::sum)));
+ return false;
+ }
+
+ private void clearBatchState(String key) {
+ retryCounts.remove(key);
+ retryNotBefore.remove(key);
+ bisectLimits.remove(key);
+ pendingFailures.remove(key);
+ }
+
+ private int bisectLimit(EventSourceMapping esm, String key) {
+ return Boolean.TRUE.equals(esm.getBisectBatchOnFunctionError())
+ ? bisectLimits.getOrDefault(key, Integer.MAX_VALUE)
+ : Integer.MAX_VALUE;
+ }
+
+ private static String batchStateKey(String uuid, String shardId, String sequence) {
+ return uuid + ":" + shardId + ":" + (sequence == null ? "TRIM_HORIZON" : sequence);
+ }
+
/**
* Returns the last record that can be consumed after a successful invocation. Floci stores the
* last consumed sequence and resumes with {@code AFTER_SEQUENCE_NUMBER}, so a partial failure
@@ -472,17 +533,18 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
return Math.min(pollIntervalMs * (1L << doublings), MAX_RETRY_BACKOFF_MS);
}
- private void sendToOnFailureDestination(EventSourceMapping esm, String shardId,
- List<DynamoDbStreamRecord> records,
- InvokeResult invokeResult,
- int invokeCount,
- String condition) {
+ /** Returns false only when the configured destination refused the batch. */
+ private boolean sendToOnFailureDestination(EventSourceMapping esm, String shardId,
+ List<DynamoDbStreamRecord> records,
+ InvokeResult invokeResult,
+ int invokeCount,
+ String condition) {
if (esm.getDestinationConfig() == null || esm.getDestinationConfig().getOnFailure() == null) {
- return;
+ return true;
}
String destinationArn = esm.getDestinationConfig().getOnFailure().getDestination();
if (destinationArn == null || destinationArn.isBlank()) {
- return;
+ return true;
}
try {
@@ -515,9 +577,11 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
esm.getUuid(), destinationArn);
}
} catch (Exception e) {
- LOG.errorv("DynamoDB Streams ESM {0}: failed to send to OnFailure destination {1}: {2}",
+ LOG.errorv("DynamoDB Streams ESM {0}: OnFailure destination {1} refused the batch, retrying the send: {2}",
esm.getUuid(), destinationArn, e.getMessage());
+ return false;
}
+ return true;
}
private String buildS3OnFailurePayload(EventSourceMapping esm, String shardId,
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
index abdcb1f7ef..96fcb8b9ea 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
@@ -33,6 +33,9 @@ import jakarta.enterprise.inject.Instance;
import jakarta.inject.Inject;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.NullSource;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor;
import java.nio.charset.StandardCharsets;
@@ -43,7 +46,11 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -1279,6 +1286,339 @@ class DynamoDbStreamsEventSourcePollerTest {
assertEquals("MaximumRecordAgeExceeded", payload.path("requestContext").path("condition").asText());
}
+ // ──────────────────── BisectBatchOnFunctionError and refused destinations ────────────────────
+
+ /**
+ * Serves the listed sequences as the stream and returns it for trimming or appending. The iterator
+ * is the sequence to read after, or empty for the trim horizon; a checkpoint missing from the stream
+ * behaves as trimmed.
+ */
+ private List<DynamoDbStreamRecord> stubStream(String... sequences) {
+ List<DynamoDbStreamRecord> stream = new CopyOnWriteArrayList<>();
+ for (String seq : sequences) {
+ stream.add(ddbRecord(seq, "INSERT", "{}"));
+ }
+ when(streamService.getShardIterator(eq(STREAM_ARN), eq(DynamoDbStreamService.SHARD_ID), anyString(), any()))
+ .thenAnswer(inv -> "TRIM_HORIZON".equals(inv.getArgument(2)) ? "" : inv.getArgument(3));
+ when(streamService.getRecords(anyString(), anyInt())).thenAnswer(inv -> {
+ String after = inv.getArgument(0);
+ List<DynamoDbStreamRecord> snapshot = new ArrayList<>(stream);
+ int from = after.isEmpty() ? 0
+ : snapshot.stream().map(DynamoDbStreamRecord::getSequenceNumber).toList().indexOf(after) + 1;
+ if (!after.isEmpty() && from == 0) {
+ throw new AwsException("TrimmedDataAccessException", "trimmed", 400);
+ }
+ int limit = inv.getArgument(1);
+ return new DynamoDbStreamService.GetRecordsResult(
+ new ArrayList<>(snapshot.subList(from, Math.min(from + limit, snapshot.size()))), "unused");
+ });
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("fn");
+ when(functionStore.getForAccount(ACCOUNT_ID, "us-east-1", "fn")).thenReturn(Optional.of(fn));
+ return stream;
+ }
+
+ /** Records the sequences of every delivered batch and answers each with {@code respond}. */
+ private List<List<String>> recordInvocations(Function<List<String>, InvokeResult> respond) {
+ List<List<String>> invocations = new CopyOnWriteArrayList<>();
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse))).thenAnswer(inv -> {
+ List<String> seqs = new ArrayList<>();
+ readRecords(inv.getArgument(1)).forEach(r -> seqs.add(r.path("dynamodb").path("SequenceNumber").asText()));
+ invocations.add(seqs);
+ return respond.apply(seqs);
+ });
+ return invocations;
+ }
+
+ private List<List<String>> failInvocationsContaining(String poison) {
+ return recordInvocations(seqs -> {
+ InvokeResult result = new InvokeResult();
+ if (seqs.contains(poison)) {
+ result.setFunctionError("Unhandled");
+ }
+ return result;
+ });
+ }
+
+ private static InvokeResult partialFailure(String sequence) {
+ InvokeResult result = new InvokeResult();
+ result.setPayload(("{\"batchItemFailures\":[{\"itemIdentifier\":\"" + sequence + "\"}]}").getBytes());
+ return result;
+ }
+
+ /** Refuses the first {@code refusals} SQS sends and returns the bodies of the accepted ones. */
+ private List<String> refuseSqsSends(int refusals) {
+ List<String> delivered = new CopyOnWriteArrayList<>();
+ AtomicInteger attempts = new AtomicInteger();
+ when(sqsService.sendMessage(anyString(), anyString(), anyInt(), anyString())).thenAnswer(inv -> {
+ if (attempts.getAndIncrement() < refusals) {
+ throw new AwsException("AWS.SimpleQueueService.NonExistentQueue", "The specified queue does not exist.", 400);
+ }
+ delivered.add(inv.getArgument(1));
+ return null;
+ });
+ return delivered;
+ }
+
+ private EventSourceMapping esmWithDlq(int maximumRetryAttempts) {
+ EventSourceMapping esm = filterEsm();
+ esm.setMaximumRetryAttempts(maximumRetryAttempts);
+ EventSourceMapping.OnFailure onFailure = new EventSourceMapping.OnFailure();
+ onFailure.setDestination("arn:aws:sqs:us-east-1:000000000000:my-dlq");
+ EventSourceMapping.DestinationConfig destinationConfig = new EventSourceMapping.DestinationConfig();
+ destinationConfig.setOnFailure(onFailure);
+ esm.setDestinationConfig(destinationConfig);
+ return esm;
+ }
+
+ private void pollOnce(DynamoDbStreamsEventSourcePoller p, EventSourceMapping esm) throws InterruptedException {
+ p.pollAndInvoke(esm);
+ awaitPollCompleted(p);
+ }
+
+ private String checkpoint(EventSourceMapping esm) {
+ return esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID);
+ }
+
+ private JsonNode batchInfo(String body) throws Exception {
+ return OBJECT_MAPPER.readTree(body).path("DDBStreamBatchInfo");
+ }
+
+ /** Also covers a trimmed checkpoint, where each poll refetches from the trim horizon. */
+ @ParameterizedTest
+ @NullSource
+ @ValueSource(strings = STALE_CHECKPOINT)
+ void bisectIsolatesThePoisonRecordAndDeliversTheRest(String startingCheckpoint) throws Exception {
+ stubStream("s1", "s2", "s3", "s4");
+ List<List<String>> invocations = failInvocationsContaining("s3");
+ EventSourceMapping esm = esmWithDlq(0);
+ esm.setBisectBatchOnFunctionError(true);
+ if (startingCheckpoint != null) {
+ esm.getShardSequenceNumbers().put(DynamoDbStreamService.SHARD_ID, startingCheckpoint);
+ }
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+
+ for (int i = 0; i < 6; i++) {
+ pollOnce(p, esm);
+ }
+
+ assertEquals(List.of(List.of("s1", "s2", "s3", "s4"), List.of("s1", "s2"), List.of("s3", "s4"),
+ List.of("s3"), List.of("s4")), invocations);
+ ArgumentCaptor<String> body = ArgumentCaptor.forClass(String.class);
+ verify(sqsService).sendMessage(anyString(), body.capture(), anyInt(), anyString());
+ assertEquals("s3", batchInfo(body.getValue()).path("startSequenceNumber").asText());
+ assertEquals("s3", batchInfo(body.getValue()).path("endSequenceNumber").asText());
+ assertEquals(1, batchInfo(body.getValue()).path("batchSize").asInt());
+ assertEquals("s4", checkpoint(esm));
+ }
+
+ @Test
+ void bisectSplitsDoNotConsumeRetryAttempts() throws Exception {
+ stubStream("s1", "s2", "s3", "s4");
+ List<List<String>> invocations = failInvocationsContaining("s3");
+ EventSourceMapping esm = esmWithDlq(1);
+ esm.setBisectBatchOnFunctionError(true);
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+
+ for (int i = 0; i < 4; i++) {
+ pollOnce(p, esm);
+ }
+ assertEquals(List.of(List.of("s1", "s2", "s3", "s4"), List.of("s1", "s2"), List.of("s3", "s4"),
+ List.of("s3")), invocations);
+ verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString());
+
+ advancePastRetry(p);
+ pollOnce(p, esm);
+
+ assertEquals(List.of("s3"), invocations.get(4));
+ ArgumentCaptor<String> body = ArgumentCaptor.forClass(String.class);
+ verify(sqsService).sendMessage(anyString(), body.capture(), anyInt(), anyString());
+ assertEquals(2, OBJECT_MAPPER.readTree(body.getValue())
+ .path("requestContext").path("approximateInvokeCount").asInt());
+ }
+
+ @Test
+ void halvedWindowFullyFilteredOutDoesNotLeakItsLimitIntoALaterHorizonRefetch() throws Exception {
+ List<DynamoDbStreamRecord> stream = stubStream("s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8");
+ stream.subList(0, 4).forEach(r -> r.setEventName("MODIFY"));
+ List<List<String>> invocations = failInvocationsContaining("s5");
+ EventSourceMapping esm = filterEsm("{\"eventName\":[\"INSERT\"]}");
+ esm.setBisectBatchOnFunctionError(true);
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+
+ pollOnce(p, esm);
+ pollOnce(p, esm);
+ assertEquals("s4", checkpoint(esm), "the halved window [s1..s4] is fully filtered out");
+ stream.removeIf(r -> "MODIFY".equals(r.getEventName()));
+ stream.add(ddbRecord("s9", "INSERT", "{}"));
+ pollOnce(p, esm);
+
+ assertEquals(List.of("s5", "s6", "s7", "s8", "s9"), invocations.get(1),
+ "the trimmed checkpoint refetches the horizon at the full BatchSize");
+ }
+
+ @Test
+ void bisectLimitNeverExceedsALaterSmallerBatchSize() throws Exception {
+ stubStream("s1", "s2", "s3", "s4");
+ List<List<String>> invocations = failInvocationsContaining("s1");
+ EventSourceMapping esm = filterEsm();
+ esm.setBisectBatchOnFunctionError(true);
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+
+ pollOnce(p, esm);
+ esm.setBatchSize(1);
+ pollOnce(p, esm);
+
+ assertEquals(List.of(List.of("s1", "s2", "s3", "s4"), List.of("s1")), invocations);
+ }
+
+ @Test
+ void turningBisectOffMidSplitRestoresTheFullBatchSize() throws Exception {
+ stubStream("s1", "s2", "s3", "s4");
+ List<List<String>> invocations = failInvocationsContaining("s1");
+ EventSourceMapping esm = filterEsm();
+ esm.setBisectBatchOnFunctionError(true);
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+
+ pollOnce(p, esm);
+ esm.setBisectBatchOnFunctionError(false);
+ advancePastRetry(p);
+ pollOnce(p, esm);
+
+ assertEquals(List.of(List.of("s1", "s2", "s3", "s4"), List.of("s1", "s2", "s3", "s4")), invocations);
+ }
+
+ @Test
+ void bisectSplitsOnFunctionErrorEvenWithReportBatchItemFailures() throws Exception {
+ stubStream("s1", "s2", "s3", "s4");
+ List<List<String>> invocations = failInvocationsContaining("s3");
+ EventSourceMapping esm = esmWithDlq(0);
+ esm.setBisectBatchOnFunctionError(true);
+ esm.setFunctionResponseTypes(List.of("ReportBatchItemFailures"));
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+
+ pollOnce(p, esm);
+ pollOnce(p, esm);
+
+ assertEquals(List.of(List.of("s1", "s2", "s3", "s4"), List.of("s1", "s2")), invocations);
+ assertEquals("s2", checkpoint(esm));
+ }
+
+ @Test
+ void partialBatchResponseWithBisectRetriesFromTheFailedRecordWithoutSplitting() throws Exception {
+ stubStream("s1", "s2", "s3", "s4");
+ List<List<String>> invocations = recordInvocations(
+ seqs -> seqs.contains("s3") ? partialFailure("s3") : new InvokeResult());
+ EventSourceMapping esm = filterEsm();
+ esm.setBisectBatchOnFunctionError(true);
+ esm.setFunctionResponseTypes(List.of("ReportBatchItemFailures"));
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+
+ pollOnce(p, esm);
+ assertEquals("s2", checkpoint(esm));
+ advancePastRetry(p);
+ pollOnce(p, esm);
+
+ assertEquals(List.of(List.of("s1", "s2", "s3", "s4"), List.of("s3", "s4")), invocations);
+ }
+
+ @Test
+ void throttledBatchIsNotBisected() throws Exception {
+ stubStream("s1", "s2", "s3", "s4");
+ AtomicBoolean throttled = new AtomicBoolean();
+ List<List<String>> invocations = recordInvocations(seqs -> {
+ if (throttled.compareAndSet(false, true)) {
+ throw new AwsException("TooManyRequestsException", "throttled", 429);
+ }
+ return new InvokeResult();
+ });
+ EventSourceMapping esm = filterEsm();
+ esm.setBisectBatchOnFunctionError(true);
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+
+ pollOnce(p, esm);
+ pollOnce(p, esm);
+
+ assertEquals(List.of(List.of("s1", "s2", "s3", "s4"), List.of("s1", "s2", "s3", "s4")), invocations);
+ assertEquals("s4", checkpoint(esm));
+ }
+
+ @Test
+ void refusedOnFailureDestinationKeepsTheBatchAndRetriesTheSend() throws Exception {
+ stubStream("s1");
+ List<List<String>> invocations = failInvocationsContaining("s1");
+ List<String> delivered = refuseSqsSends(1);
+ EventSourceMapping esm = esmWithDlq(0);
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+
+ pollOnce(p, esm);
+ assertNull(checkpoint(esm), "a refused send must not checkpoint the discarded batch");
+ pollOnce(p, esm);
+ verify(sqsService, times(1)).sendMessage(anyString(), anyString(), anyInt(), anyString());
+
+ advancePastRetry(p);
+ pollOnce(p, esm);
+
+ assertEquals(List.of(List.of("s1")), invocations, "the parked batch is re-sent, not re-invoked");
+ assertEquals(1, delivered.size());
+ assertEquals("s1", batchInfo(delivered.get(0)).path("startSequenceNumber").asText());
+ assertEquals("s1", batchInfo(delivered.get(0)).path("endSequenceNumber").asText());
+ assertEquals("s1", checkpoint(esm));
+ }
+
+ @Test
+ void parkedFailureIsResentAfterTheFunctionStopsResolving() throws Exception {
+ stubStream("s1");
+ failInvocationsContaining("s1");
+ List<String> delivered = refuseSqsSends(1);
+ EventSourceMapping esm = esmWithDlq(0);
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+
+ pollOnce(p, esm);
+ assertNull(checkpoint(esm));
+ when(functionStore.getForAccount(ACCOUNT_ID, "us-east-1", "fn")).thenReturn(Optional.empty());
+ advancePastRetry(p);
+ pollOnce(p, esm);
+
+ assertEquals(1, delivered.size(), "the resend does not need the function");
+ assertEquals("s1", checkpoint(esm));
+ }
+
+ @Test
+ void refusedOnFailureDestinationAfterPartialSuccessKeepsTheAcknowledgedPrefix() throws Exception {
+ stubStream("s1", "s2", "s3");
+ List<List<String>> invocations = recordInvocations(seqs -> partialFailure("s2"));
+ List<String> delivered = refuseSqsSends(1);
+ EventSourceMapping esm = esmWithDlq(0);
+ esm.setFunctionResponseTypes(List.of("ReportBatchItemFailures"));
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+
+ pollOnce(p, esm);
+ assertEquals("s1", checkpoint(esm));
+
+ advancePastRetry(p);
+ pollOnce(p, esm);
+
+ assertEquals(1, invocations.size());
+ assertEquals(1, delivered.size());
+ assertEquals("s2", batchInfo(delivered.get(0)).path("startSequenceNumber").asText());
+ assertEquals("s3", batchInfo(delivered.get(0)).path("endSequenceNumber").asText());
+ assertEquals("s3", checkpoint(esm));
+ }
+
+ @Test
+ void exhaustedBatchWithoutOnFailureDestinationIsDiscarded() throws Exception {
+ stubStream("s1");
+ failInvocationsContaining("s1");
+ EventSourceMapping esm = filterEsm();
+ esm.setMaximumRetryAttempts(0);
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+
+ pollOnce(p, esm);
+
+ assertEquals("s1", checkpoint(esm));
+ }
+
private void awaitPollCompleted(DynamoDbStreamsEventSourcePoller poller) throws InterruptedException {
long deadline = System.currentTimeMillis() + 3_000;
while (!poller.activePolls.isEmpty() && System.currentTimeMillis() < deadline) {