Light, fluffy, and always free - The AWS Local Emulator alternative
feat(firehose): retry failed Redshift COPY loads every five minutes (#6111)
Vũ Thế Huy99ef36998ca1
diff --git a/docs/services/firehose.md b/docs/services/firehose.mdindex 0bf758ef3d..4be6366b78 100644--- a/docs/services/firehose.md+++ b/docs/services/firehose.md@@ -158,11 +158,14 @@ or from the secret in `SecretsManagerConfiguration` when `Enabled` is true (the(an unquoted name is lowercased, a quoted one is kept).A batch is staged in S3 once, and the `COPY` runs on its own thread so that a slow or hanging query does not hold the-buffer flusher thread or `PutRecord`. If its `COPY` fails (missing table or cluster, bad data), the failure is logged-and the staged object stays in S3; retrying failed loads will follow in a separate change. With `S3BackupMode` set to-`Enabled`, which needs an `S3BackupConfiguration`, each batch is also written once to that bucket under its own prefix-and compression. The backup is written after the batch is staged, and a failed backup write is logged without holding-the load back or re-staging the batch, so that batch then has no backup copy.+buffer flusher thread or `PutRecord`. If its `COPY` fails (missing table or cluster, bad data), the staged object stays in S3+and the `COPY` is retried against that same object every 5 minutes until `RetryOptions.DurationInSeconds` has elapsed (the+window starts with the first attempt, default 3600, `0` means a single attempt). Only the first failure of a batch is logged+as a warning. Batches are loaded in order, so a newer batch waits behind an older failing one. After the window the batch is+given up with an error log and the staged object is kept. Pending copies are held in memory and are lost on restart. With `S3BackupMode`+set to `Enabled`, which needs an `S3BackupConfiguration`, each batch is also written once to that bucket under its own prefix+and compression. The backup is written after the batch is staged and queued, and a failed backup write is logged without+holding the load back or re-staging the batch, so that batch then has no backup copy.A JDBC connection that stops answering is dropped after 150 seconds.diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.javaindex 14dbd9af48..40dd293f21 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java@@ -35,12 +35,14 @@ import java.time.Clock;import java.time.Instant;import java.util.ArrayList;import java.util.Collections;+import java.util.Deque;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.Optional;import java.util.Set;import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.ConcurrentLinkedDeque;import java.util.concurrent.Executor;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;@@ -54,6 +56,9 @@ public class FirehoseService implements ResourceProvider {private static final Logger LOG = Logger.getLogger(FirehoseService.class);private static final String DEFAULT_BUCKET = "floci-firehose-results";private static final int DEFAULT_BUFFERING_INTERVAL_SECONDS = 300;+ private static final int DEFAULT_REDSHIFT_RETRY_SECONDS = 3600;+ // AWS retries a failed Redshift load every 5 minutes.+ private static final long REDSHIFT_RETRY_INTERVAL_SECONDS = 300;private static final int DEFAULT_BUFFERING_SIZE_MBS = 5;/*** botocore's {@code AWSKMSKeyARNForSSE} shape: a KMS <em>key</em> ARN, not an alias and@@ -119,6 +124,11 @@ public class FirehoseService implements ResourceProvider {private final FirehoseLambdaTransformer lambdaTransformer;private final FirehoseRedshiftDelivery redshiftDelivery;private final Executor redshiftDeliveryExecutor;+ // Per stream, the batches staged in S3 whose COPY has not succeeded yet, oldest first. A failed COPY is+ // retried against the object already staged instead of staging the batch again.+ private final Map<String, Deque<PendingRedshiftCopy>> redshiftPending = new ConcurrentHashMap<>();+ // Tracks streams that currently have a drain task queued or running on redshiftDeliveryExecutor.+ private final Set<String> activeRedshiftDrains = ConcurrentHashMap.newKeySet();// Serialises the read, version check, merge and write of a Redshift UpdateDestination per stream.private final Map<String, Object> redshiftUpdateLocks = new ConcurrentHashMap<>();private final Clock clock;@@ -306,6 +316,7 @@ public class FirehoseService implements ResourceProvider {try {pollKinesisSources();flushDueBuffers(clock.instant());+ retryPendingRedshiftCopies();} catch (Throwable t) {LOG.warnv("Firehose buffer flush tick failed: {0}", t.getMessage());}@@ -840,6 +851,8 @@ public class FirehoseService implements ResourceProvider {// reached the bucket after DeleteDeliveryStream completed.buffers.remove(streamKey);bufferSince.remove(streamKey);+ redshiftPending.remove(streamKey);+ activeRedshiftDrains.remove(streamKey);pendingSourceIterators.remove(streamKey);iteratorDelete(streamKey);LOG.infov("Deleted Firehose delivery stream: {0}", name);@@ -1150,10 +1163,9 @@ public class FirehoseService implements ResourceProvider {}/**- * Everything after a Redshift batch is staged in S3. The batch is durable in S3, so no failure here may hand its- * records back to the buffer: that would stage and load them a second time. Backups and source checkpoints are- * completed first, then COPY is submitted to run on its own thread so that a slow or hanging query never holds- * the buffer flusher thread or PutRecord.+ * Everything after a Redshift batch is staged and queued. The batch is durable from the moment it is queued, so+ * no step here may hand its records back to the buffer: that would stage and load them a second time. A failing+ * backup write or checkpoint write is logged instead, and the load itself retries from the queue.*/private void deliverRedshiftBatch(String streamName, DeliveryStreamDescription stream,String bucket, String key, byte[] rawPayload,@@ -1171,29 +1183,154 @@ public class FirehoseService implements ResourceProvider {LOG.errorv(e, "Could not persist the source checkpoint of stream {0} after its batch was staged; the "+ "source records may be read again: {1}", streamName, describe(e));}- if (redshiftDelivery == null) {- LOG.warnv("Stream {0} has a Redshift destination but Redshift delivery is unavailable; "- + "the batch stays in s3://{1}/{2}", streamName, bucket, key);+ redshiftPending.compute(streamName, (k, existing) -> {+ Deque<PendingRedshiftCopy> queue = existing != null ? existing : new ConcurrentLinkedDeque<>();+ queue.addLast(new PendingRedshiftCopy(bucket, key));+ return queue;+ });+ submitRedshiftDrain(streamName, stream);+ }++ /**+ * A batch already staged in S3 and waiting for its COPY. {@code failingSince} marks when the initial attempt began+ * and {@code nextAttempt} spaces the retries out by 5 minutes, so a failing load neither floods the log nor holds the+ * shared tick thread for a connection attempt on every tick.+ */+ private static final class PendingRedshiftCopy {+ private final String bucket;+ private final String key;+ private Instant failingSince;+ private Instant nextAttempt;+ private int failures;++ private PendingRedshiftCopy(String bucket, String key) {+ this.bucket = bucket;+ this.key = key;+ }+ }++ /** Retries the COPY of every stream that still has staged batches waiting and due. Called on each tick. */+ void retryPendingRedshiftCopies() {+ for (String streamKey : redshiftPending.keySet()) {+ if (!isRedshiftDrainDue(streamKey)) {+ continue;+ }+ Optional<DeliveryStreamDescription> stream = streamGet(streamKey);+ if (stream.isEmpty() || !isRedshiftStream(stream.get())) {+ redshiftPending.remove(streamKey);+ continue;+ }+ submitRedshiftDrain(streamKey, stream.get());+ }+ }++ private boolean isRedshiftDrainDue(String streamKey) {+ Deque<PendingRedshiftCopy> queue = redshiftPending.get(streamKey);+ if (queue == null || queue.isEmpty()) {+ return false;+ }+ PendingRedshiftCopy head = queue.peekFirst();+ return head != null && (head.nextAttempt == null || !clock.instant().isBefore(head.nextAttempt));+ }++ private void submitRedshiftDrain(String streamKey, DeliveryStreamDescription stream) {+ if (!activeRedshiftDrains.add(streamKey)) {return;}try {redshiftDeliveryExecutor.execute(() -> {- RequestScopes.runAs(stream.getAccountId(), () -> {- try {- String region = AwsArnUtils.regionOrDefault(stream.getDeliveryStreamARN(),- regionResolver.getDefaultRegion());- redshiftDelivery.copy(stream.getAccountId(), region, redshift, bucket, key);- } catch (Exception e) {- LOG.errorv("Redshift delivery of s3://{0}/{1} into table {2} failed for stream {3}; "- + "the batch stays in s3://{0}/{1}: {4}", bucket, key,- redshift.getCopyCommand().getDataTableName(), streamName, describe(e));+ try {+ RequestScopes.runAs(stream.getAccountId(), () -> drainRedshiftCopies(streamKey, stream));+ } finally {+ activeRedshiftDrains.remove(streamKey);+ if (redshiftPending.containsKey(streamKey) && isRedshiftDrainDue(streamKey)) {+ submitRedshiftDrain(streamKey, stream);}- });+ }});} catch (RejectedExecutionException e) {- LOG.errorv("Redshift delivery executor rejected task for s3://{0}/{1} into table {2} on stream {3}; "- + "the batch stays in s3://{0}/{1}: {4}", bucket, key,- redshift.getCopyCommand().getDataTableName(), streamName, describe(e));+ activeRedshiftDrains.remove(streamKey);+ LOG.errorv("Redshift delivery executor rejected task for stream {0}: {1}", streamKey, describe(e));+ }+ }++ /** Loads the stream's staged batches oldest first and stops at the first one that must be retried. */+ private void drainRedshiftCopies(String streamKey, DeliveryStreamDescription stream) {+ Deque<PendingRedshiftCopy> queue = redshiftPending.get(streamKey);+ if (queue == null) {+ return;+ }+ if (!isRedshiftStream(stream)) {+ redshiftPending.remove(streamKey);+ return;+ }+ RedshiftDestination redshift = stream.getDestinations().get(0).getRedshiftDestinationDescription();+ PendingRedshiftCopy next;+ while (redshiftPending.containsKey(streamKey) && (next = queue.peekFirst()) != null+ && copyStaged(streamKey, stream, redshift, next)) {+ queue.pollFirst();+ }+ redshiftPending.computeIfPresent(streamKey, (k, remaining) -> remaining.isEmpty() ? null : remaining);+ }++ /**+ * Returns true when the batch is accounted for (loaded, or failed past the retry window) and+ * false when it must be tried again.+ */+ private boolean copyStaged(String streamKey, DeliveryStreamDescription stream, RedshiftDestination redshift,+ PendingRedshiftCopy pending) {+ String table = redshift.getCopyCommand().getDataTableName();+ if (redshiftDelivery == null) {+ LOG.warnv("Stream {0} has a Redshift destination but Redshift delivery is unavailable; "+ + "the batch stays in s3://{1}/{2}", streamKey, pending.bucket, pending.key);+ return true;+ }+ Instant now = clock.instant();+ if (pending.nextAttempt != null && now.isBefore(pending.nextAttempt)) {+ return false;+ }+ int window = redshift.getRetryOptions() != null && redshift.getRetryOptions().getDurationInSeconds() != null+ ? redshift.getRetryOptions().getDurationInSeconds() : DEFAULT_REDSHIFT_RETRY_SECONDS;+ if (pending.failingSince != null && window > 0 && !now.isBefore(pending.failingSince.plusSeconds(window))) {+ LOG.errorv("Redshift delivery into table {0} failed for stream {1} and the retry window of {2}s is over; "+ + "the batch stays in s3://{3}/{4}", table, streamKey, window, pending.bucket, pending.key);+ return true;+ }+ Instant attemptStart = now;+ try {+ String region = AwsArnUtils.regionOrDefault(stream.getDeliveryStreamARN(),+ regionResolver.getDefaultRegion());+ redshiftDelivery.copy(stream.getAccountId(), region, redshift, pending.bucket, pending.key);+ return true;+ } catch (RuntimeException e) {+ Instant failedAt = clock.instant();+ boolean expected = e instanceof AwsException || e instanceof IllegalStateException+ || e instanceof IllegalArgumentException;+ if (pending.failingSince == null) {+ pending.failingSince = attemptStart;+ }+ pending.failures++;+ if (window > 0 && failedAt.isBefore(pending.failingSince.plusSeconds(window))) {+ pending.nextAttempt = failedAt.plusSeconds(REDSHIFT_RETRY_INTERVAL_SECONDS);+ String reason = describe(e);+ if (!expected) {+ LOG.errorv(e, "Unexpected failure delivering s3://{0}/{1} into table {2} for stream {3}; "+ + "retrying in {4}s", pending.bucket, pending.key, table, streamKey, REDSHIFT_RETRY_INTERVAL_SECONDS);+ } else if (pending.failures == 1) {+ LOG.warnv("Redshift delivery of s3://{0}/{1} into table {2} failed for stream {3}, "+ + "retrying in {4}s: {5}", pending.bucket, pending.key, table, streamKey, REDSHIFT_RETRY_INTERVAL_SECONDS,+ reason);+ } else {+ LOG.debugv("Redshift delivery of s3://{0}/{1} into table {2} failed again for stream {3} "+ + "(attempt {4}), retrying in {5}s: {6}", pending.bucket, pending.key, table, streamKey,+ pending.failures, REDSHIFT_RETRY_INTERVAL_SECONDS, reason);+ }+ return false;+ }+ LOG.errorv("Redshift delivery into table {0} failed for stream {1} and the retry window of {2}s is over; "+ + "the batch stays in s3://{3}/{4}: {5}", table, streamKey, window, pending.bucket, pending.key,+ describe(e));+ return true;}}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.javaindex b21dd10bf6..ee08e0b71a 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java@@ -17,7 +17,9 @@ import org.mockito.InOrder;import org.mockito.Mockito;import java.nio.charset.StandardCharsets;+import java.time.Duration;import java.util.HashMap;+import java.util.HashSet;import java.util.List;import java.util.Map;import java.util.concurrent.Executor;@@ -26,6 +28,7 @@ import java.util.concurrent.Executors;import java.util.concurrent.RejectedExecutionException;import java.util.concurrent.TimeUnit;+import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyLong;@@ -104,12 +107,19 @@ class FirehoseRedshiftServiceTest {anyMap(), any());}+ /** AWS retries a failed Redshift load every 5 minutes, so 301 seconds is always past it. */+ private void pastBackoff() {+ clock.advance(Duration.ofSeconds(301));+ }+@Testvoid successfulCopyConsumesTheBatchOnce() throws Exception {createStream("");service.flush(STREAM);service.flush(STREAM);+ pastBackoff();+ service.retryPendingRedshiftCopies();verify(delivery, times(1)).copy(eq("000000000000"), eq("us-east-1"), any(), eq("stage"), anyString());verifyStagedTimes(1);@@ -276,4 +286,234 @@ class FirehoseRedshiftServiceTest {inOrder.verify(mockExecutor).awaitTermination(5, TimeUnit.SECONDS);inOrder.verify(mockExecutor).shutdownNow();}++ @Test+ void failedCopyIsRetriedAgainstTheSameStagedObjectWithoutRestaging() throws Exception {+ createStream("");+ copyFails();++ service.flush(STREAM);+ pastBackoff();+ service.retryPendingRedshiftCopies();+ pastBackoff();+ service.retryPendingRedshiftCopies();++ ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);+ verify(delivery, times(3)).copy(anyString(), anyString(), any(), eq("stage"), keys.capture());+ assertEquals(1, new HashSet<>(keys.getAllValues()).size(), "every attempt must use the one staged key");+ verifyStagedTimes(1);+ }++ @Test+ void failedCopyIsNotRetriedBeforeTheBackoffHasElapsed() throws Exception {+ createStream("");+ copyFails();++ service.flush(STREAM);+ service.retryPendingRedshiftCopies();+ service.retryPendingRedshiftCopies();++ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());+ }++ @Test+ void theSchedulerTickRetriesPendingCopies() throws Exception {+ createStream("");+ copyFails();+ service.flush(STREAM);+ pastBackoff();++ service.tickSafely();++ verify(delivery, times(2)).copy(anyString(), anyString(), any(), anyString(), anyString());+ }++ @Test+ void recoveredCopyStopsBeingRetried() throws Exception {+ createStream("");+ copyFails();+ service.flush(STREAM);+ Mockito.reset(delivery);++ pastBackoff();+ service.retryPendingRedshiftCopies();+ pastBackoff();+ service.retryPendingRedshiftCopies();++ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());+ }++ @Test+ void failedCopyIsDroppedAfterTheRetryWindowButTheStagedObjectStays() throws Exception {+ createStream(", \"RetryOptions\": {\"DurationInSeconds\": 350}");+ copyFails();++ service.flush(STREAM);+ pastBackoff();+ service.retryPendingRedshiftCopies();+ pastBackoff();+ service.retryPendingRedshiftCopies();++ verify(delivery, times(2)).copy(anyString(), anyString(), any(), anyString(), anyString());+ verifyStagedTimes(1);+ }++ @Test+ void retryWindowShorterThanBackoffExpiresWithoutRetrying() throws Exception {+ createStream(", \"RetryOptions\": {\"DurationInSeconds\": 10}");+ copyFails();++ service.flush(STREAM);+ pastBackoff();+ service.retryPendingRedshiftCopies();++ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());+ verifyStagedTimes(1);+ }++ @Test+ void slowFailureMeasuresNextAttemptFromFailureTime() throws Exception {+ createStream("");+ Mockito.doAnswer(invocation -> {+ clock.advance(Duration.ofSeconds(150));+ throw new IllegalStateException("slow connection failure");+ }).when(delivery).copy(anyString(), anyString(), any(), anyString(), anyString());++ service.flush(STREAM);+ clock.advance(Duration.ofSeconds(151));+ service.retryPendingRedshiftCopies();+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());++ clock.advance(Duration.ofSeconds(150));+ service.retryPendingRedshiftCopies();+ verify(delivery, times(2)).copy(anyString(), anyString(), any(), anyString(), anyString());+ }++ @Test+ void firstAttemptTakingLongerThanRetryWindowDoesNotRetry() throws Exception {+ createStream(", \"RetryOptions\": {\"DurationInSeconds\": 100}");+ Mockito.doAnswer(invocation -> {+ clock.advance(Duration.ofSeconds(150));+ throw new IllegalStateException("long initial attempt failed");+ }).when(delivery).copy(anyString(), anyString(), any(), anyString(), anyString());++ service.flush(STREAM);+ pastBackoff();+ service.retryPendingRedshiftCopies();++ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());+ verifyStagedTimes(1);+ }++ @Test+ void zeroRetryDurationMeansASingleAttempt() throws Exception {+ createStream(", \"RetryOptions\": {\"DurationInSeconds\": 0}");+ copyFails();++ service.flush(STREAM);+ pastBackoff();+ service.retryPendingRedshiftCopies();++ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());+ }++ @Test+ void anUnexpectedFailureDoesNotEscapeAndIsRetried() throws Exception {+ createStream("");+ doThrow(new NullPointerException("unexpected internal bug")).when(delivery)+ .copy(anyString(), anyString(), any(), anyString(), anyString());++ service.flush(STREAM);+ pastBackoff();+ service.retryPendingRedshiftCopies();++ verify(delivery, times(2)).copy(anyString(), anyString(), any(), anyString(), anyString());+ }++ @Test+ void aNewerBatchWaitsBehindAnOlderFailingOne() throws Exception {+ createStream("");+ copyFails();+ service.flush(STREAM);+ service.putRecord(STREAM, new Record("{\"id\":2}".getBytes(StandardCharsets.UTF_8)));++ service.flush(STREAM);++ ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), eq("stage"), keys.capture());+ verifyStagedTimes(2);+ }++ @Test+ void deletingTheStreamDiscardsPendingCopies() throws Exception {+ createStream("");+ copyFails();+ service.flush(STREAM);++ service.deleteDeliveryStream(STREAM);+ pastBackoff();+ service.retryPendingRedshiftCopies();++ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());+ }++ @Test+ void deletingStreamWhileDrainIteratesStopsFurtherCopies() throws Exception {+ createStream("");+ copyFails();+ service.flush(STREAM);+ service.putRecord(STREAM, new Record("{\"id\":2}".getBytes(StandardCharsets.UTF_8)));+ service.flush(STREAM);++ Mockito.reset(delivery);+ Mockito.doAnswer(invocation -> {+ service.deleteDeliveryStream(STREAM);+ return null;+ }).when(delivery).copy(anyString(), anyString(), any(), anyString(), anyString());++ pastBackoff();+ service.retryPendingRedshiftCopies();++ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());+ }++ @Test+ void retryPendingRedshiftCopiesDoesNotRunWhenHeadIsNotDue() throws Exception {+ createStream("");+ copyFails();++ service.flush(STREAM);+ for (int i = 0; i < 5; i++) {+ service.retryPendingRedshiftCopies();+ }++ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());+ }++ @Test+ void pendingCopiesAreDiscardedIfDestinationIsNoLongerRedshift() throws Exception {+ createStream("");+ copyFails();+ service.flush(STREAM);++ service.describeDeliveryStream(STREAM).getDestinations().get(0).setRedshiftDestinationDescription(null);++ pastBackoff();+ service.retryPendingRedshiftCopies();++ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());+ }++ @Test+ void backupEnabledWritesTheBatchToTheBackupBucketOnlyOnce() throws Exception {+ createStream(BACKUP_ENABLED);+ copyFails();++ service.flush(STREAM);+ pastBackoff();+ service.retryPendingRedshiftCopies();++ verifyStagedTimes(1);+ verify(s3Service, times(1)).putObject(eq("bk"), anyString(), any(byte[].class), anyString(), anyMap(), any());+ }}