Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(firehose): retry failed Redshift COPY loads every five minutes (#6111)

Vũ Thế Huy99ef36998ca1
diff --git a/docs/services/firehose.md b/docs/services/firehose.md
index 0bf758ef3d..4be6366b78 100644
--- a/docs/services/firehose.md
+++ b/docs/services/firehose.md
@@ -158,11 +158,14 @@ or from the secret in `SecretsManagerConfiguration` when `Enabled` is true (the
(an unquoted name is lowercased, a quoted one is kept).
A batch is staged in S3 once, and the `COPY` runs on its own thread so that a slow or hanging query does not hold the
-buffer flusher thread or `PutRecord`. If its `COPY` fails (missing table or cluster, bad data), the failure is logged
-and the staged object stays in S3; retrying failed loads will follow in a separate change. With `S3BackupMode` set to
-`Enabled`, which needs an `S3BackupConfiguration`, each batch is also written once to that bucket under its own prefix
-and compression. The backup is written after the batch is staged, and a failed backup write is logged without holding
-the load back or re-staging the batch, so that batch then has no backup copy.
+buffer flusher thread or `PutRecord`. If its `COPY` fails (missing table or cluster, bad data), the staged object stays in S3
+and the `COPY` is retried against that same object every 5 minutes until `RetryOptions.DurationInSeconds` has elapsed (the
+window starts with the first attempt, default 3600, `0` means a single attempt). Only the first failure of a batch is logged
+as a warning. Batches are loaded in order, so a newer batch waits behind an older failing one. After the window the batch is
+given up with an error log and the staged object is kept. Pending copies are held in memory and are lost on restart. With `S3BackupMode`
+set to `Enabled`, which needs an `S3BackupConfiguration`, each batch is also written once to that bucket under its own prefix
+and compression. The backup is written after the batch is staged and queued, and a failed backup write is logged without
+holding the load back or re-staging the batch, so that batch then has no backup copy.
A JDBC connection that stops answering is dropped after 150 seconds.
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
index 14dbd9af48..40dd293f21 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
@@ -35,12 +35,14 @@ import java.time.Clock;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Deque;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -54,6 +56,9 @@ public class FirehoseService implements ResourceProvider {
private static final Logger LOG = Logger.getLogger(FirehoseService.class);
private static final String DEFAULT_BUCKET = "floci-firehose-results";
private static final int DEFAULT_BUFFERING_INTERVAL_SECONDS = 300;
+ private static final int DEFAULT_REDSHIFT_RETRY_SECONDS = 3600;
+ // AWS retries a failed Redshift load every 5 minutes.
+ private static final long REDSHIFT_RETRY_INTERVAL_SECONDS = 300;
private static final int DEFAULT_BUFFERING_SIZE_MBS = 5;
/**
* botocore's {@code AWSKMSKeyARNForSSE} shape: a KMS <em>key</em> ARN, not an alias and
@@ -119,6 +124,11 @@ public class FirehoseService implements ResourceProvider {
private final FirehoseLambdaTransformer lambdaTransformer;
private final FirehoseRedshiftDelivery redshiftDelivery;
private final Executor redshiftDeliveryExecutor;
+ // Per stream, the batches staged in S3 whose COPY has not succeeded yet, oldest first. A failed COPY is
+ // retried against the object already staged instead of staging the batch again.
+ private final Map<String, Deque<PendingRedshiftCopy>> redshiftPending = new ConcurrentHashMap<>();
+ // Tracks streams that currently have a drain task queued or running on redshiftDeliveryExecutor.
+ private final Set<String> activeRedshiftDrains = ConcurrentHashMap.newKeySet();
// Serialises the read, version check, merge and write of a Redshift UpdateDestination per stream.
private final Map<String, Object> redshiftUpdateLocks = new ConcurrentHashMap<>();
private final Clock clock;
@@ -306,6 +316,7 @@ public class FirehoseService implements ResourceProvider {
try {
pollKinesisSources();
flushDueBuffers(clock.instant());
+ retryPendingRedshiftCopies();
} catch (Throwable t) {
LOG.warnv("Firehose buffer flush tick failed: {0}", t.getMessage());
}
@@ -840,6 +851,8 @@ public class FirehoseService implements ResourceProvider {
// reached the bucket after DeleteDeliveryStream completed.
buffers.remove(streamKey);
bufferSince.remove(streamKey);
+ redshiftPending.remove(streamKey);
+ activeRedshiftDrains.remove(streamKey);
pendingSourceIterators.remove(streamKey);
iteratorDelete(streamKey);
LOG.infov("Deleted Firehose delivery stream: {0}", name);
@@ -1150,10 +1163,9 @@ public class FirehoseService implements ResourceProvider {
}
/**
- * Everything after a Redshift batch is staged in S3. The batch is durable in S3, so no failure here may hand its
- * records back to the buffer: that would stage and load them a second time. Backups and source checkpoints are
- * completed first, then COPY is submitted to run on its own thread so that a slow or hanging query never holds
- * the buffer flusher thread or PutRecord.
+ * Everything after a Redshift batch is staged and queued. The batch is durable from the moment it is queued, so
+ * no step here may hand its records back to the buffer: that would stage and load them a second time. A failing
+ * backup write or checkpoint write is logged instead, and the load itself retries from the queue.
*/
private void deliverRedshiftBatch(String streamName, DeliveryStreamDescription stream,
String bucket, String key, byte[] rawPayload,
@@ -1171,29 +1183,154 @@ public class FirehoseService implements ResourceProvider {
LOG.errorv(e, "Could not persist the source checkpoint of stream {0} after its batch was staged; the "
+ "source records may be read again: {1}", streamName, describe(e));
}
- if (redshiftDelivery == null) {
- LOG.warnv("Stream {0} has a Redshift destination but Redshift delivery is unavailable; "
- + "the batch stays in s3://{1}/{2}", streamName, bucket, key);
+ redshiftPending.compute(streamName, (k, existing) -> {
+ Deque<PendingRedshiftCopy> queue = existing != null ? existing : new ConcurrentLinkedDeque<>();
+ queue.addLast(new PendingRedshiftCopy(bucket, key));
+ return queue;
+ });
+ submitRedshiftDrain(streamName, stream);
+ }
+
+ /**
+ * A batch already staged in S3 and waiting for its COPY. {@code failingSince} marks when the initial attempt began
+ * and {@code nextAttempt} spaces the retries out by 5 minutes, so a failing load neither floods the log nor holds the
+ * shared tick thread for a connection attempt on every tick.
+ */
+ private static final class PendingRedshiftCopy {
+ private final String bucket;
+ private final String key;
+ private Instant failingSince;
+ private Instant nextAttempt;
+ private int failures;
+
+ private PendingRedshiftCopy(String bucket, String key) {
+ this.bucket = bucket;
+ this.key = key;
+ }
+ }
+
+ /** Retries the COPY of every stream that still has staged batches waiting and due. Called on each tick. */
+ void retryPendingRedshiftCopies() {
+ for (String streamKey : redshiftPending.keySet()) {
+ if (!isRedshiftDrainDue(streamKey)) {
+ continue;
+ }
+ Optional<DeliveryStreamDescription> stream = streamGet(streamKey);
+ if (stream.isEmpty() || !isRedshiftStream(stream.get())) {
+ redshiftPending.remove(streamKey);
+ continue;
+ }
+ submitRedshiftDrain(streamKey, stream.get());
+ }
+ }
+
+ private boolean isRedshiftDrainDue(String streamKey) {
+ Deque<PendingRedshiftCopy> queue = redshiftPending.get(streamKey);
+ if (queue == null || queue.isEmpty()) {
+ return false;
+ }
+ PendingRedshiftCopy head = queue.peekFirst();
+ return head != null && (head.nextAttempt == null || !clock.instant().isBefore(head.nextAttempt));
+ }
+
+ private void submitRedshiftDrain(String streamKey, DeliveryStreamDescription stream) {
+ if (!activeRedshiftDrains.add(streamKey)) {
return;
}
try {
redshiftDeliveryExecutor.execute(() -> {
- RequestScopes.runAs(stream.getAccountId(), () -> {
- try {
- String region = AwsArnUtils.regionOrDefault(stream.getDeliveryStreamARN(),
- regionResolver.getDefaultRegion());
- redshiftDelivery.copy(stream.getAccountId(), region, redshift, bucket, key);
- } catch (Exception e) {
- LOG.errorv("Redshift delivery of s3://{0}/{1} into table {2} failed for stream {3}; "
- + "the batch stays in s3://{0}/{1}: {4}", bucket, key,
- redshift.getCopyCommand().getDataTableName(), streamName, describe(e));
+ try {
+ RequestScopes.runAs(stream.getAccountId(), () -> drainRedshiftCopies(streamKey, stream));
+ } finally {
+ activeRedshiftDrains.remove(streamKey);
+ if (redshiftPending.containsKey(streamKey) && isRedshiftDrainDue(streamKey)) {
+ submitRedshiftDrain(streamKey, stream);
}
- });
+ }
});
} catch (RejectedExecutionException e) {
- LOG.errorv("Redshift delivery executor rejected task for s3://{0}/{1} into table {2} on stream {3}; "
- + "the batch stays in s3://{0}/{1}: {4}", bucket, key,
- redshift.getCopyCommand().getDataTableName(), streamName, describe(e));
+ activeRedshiftDrains.remove(streamKey);
+ LOG.errorv("Redshift delivery executor rejected task for stream {0}: {1}", streamKey, describe(e));
+ }
+ }
+
+ /** Loads the stream's staged batches oldest first and stops at the first one that must be retried. */
+ private void drainRedshiftCopies(String streamKey, DeliveryStreamDescription stream) {
+ Deque<PendingRedshiftCopy> queue = redshiftPending.get(streamKey);
+ if (queue == null) {
+ return;
+ }
+ if (!isRedshiftStream(stream)) {
+ redshiftPending.remove(streamKey);
+ return;
+ }
+ RedshiftDestination redshift = stream.getDestinations().get(0).getRedshiftDestinationDescription();
+ PendingRedshiftCopy next;
+ while (redshiftPending.containsKey(streamKey) && (next = queue.peekFirst()) != null
+ && copyStaged(streamKey, stream, redshift, next)) {
+ queue.pollFirst();
+ }
+ redshiftPending.computeIfPresent(streamKey, (k, remaining) -> remaining.isEmpty() ? null : remaining);
+ }
+
+ /**
+ * Returns true when the batch is accounted for (loaded, or failed past the retry window) and
+ * false when it must be tried again.
+ */
+ private boolean copyStaged(String streamKey, DeliveryStreamDescription stream, RedshiftDestination redshift,
+ PendingRedshiftCopy pending) {
+ String table = redshift.getCopyCommand().getDataTableName();
+ if (redshiftDelivery == null) {
+ LOG.warnv("Stream {0} has a Redshift destination but Redshift delivery is unavailable; "
+ + "the batch stays in s3://{1}/{2}", streamKey, pending.bucket, pending.key);
+ return true;
+ }
+ Instant now = clock.instant();
+ if (pending.nextAttempt != null && now.isBefore(pending.nextAttempt)) {
+ return false;
+ }
+ int window = redshift.getRetryOptions() != null && redshift.getRetryOptions().getDurationInSeconds() != null
+ ? redshift.getRetryOptions().getDurationInSeconds() : DEFAULT_REDSHIFT_RETRY_SECONDS;
+ if (pending.failingSince != null && window > 0 && !now.isBefore(pending.failingSince.plusSeconds(window))) {
+ LOG.errorv("Redshift delivery into table {0} failed for stream {1} and the retry window of {2}s is over; "
+ + "the batch stays in s3://{3}/{4}", table, streamKey, window, pending.bucket, pending.key);
+ return true;
+ }
+ Instant attemptStart = now;
+ try {
+ String region = AwsArnUtils.regionOrDefault(stream.getDeliveryStreamARN(),
+ regionResolver.getDefaultRegion());
+ redshiftDelivery.copy(stream.getAccountId(), region, redshift, pending.bucket, pending.key);
+ return true;
+ } catch (RuntimeException e) {
+ Instant failedAt = clock.instant();
+ boolean expected = e instanceof AwsException || e instanceof IllegalStateException
+ || e instanceof IllegalArgumentException;
+ if (pending.failingSince == null) {
+ pending.failingSince = attemptStart;
+ }
+ pending.failures++;
+ if (window > 0 && failedAt.isBefore(pending.failingSince.plusSeconds(window))) {
+ pending.nextAttempt = failedAt.plusSeconds(REDSHIFT_RETRY_INTERVAL_SECONDS);
+ String reason = describe(e);
+ if (!expected) {
+ LOG.errorv(e, "Unexpected failure delivering s3://{0}/{1} into table {2} for stream {3}; "
+ + "retrying in {4}s", pending.bucket, pending.key, table, streamKey, REDSHIFT_RETRY_INTERVAL_SECONDS);
+ } else if (pending.failures == 1) {
+ LOG.warnv("Redshift delivery of s3://{0}/{1} into table {2} failed for stream {3}, "
+ + "retrying in {4}s: {5}", pending.bucket, pending.key, table, streamKey, REDSHIFT_RETRY_INTERVAL_SECONDS,
+ reason);
+ } else {
+ LOG.debugv("Redshift delivery of s3://{0}/{1} into table {2} failed again for stream {3} "
+ + "(attempt {4}), retrying in {5}s: {6}", pending.bucket, pending.key, table, streamKey,
+ pending.failures, REDSHIFT_RETRY_INTERVAL_SECONDS, reason);
+ }
+ return false;
+ }
+ LOG.errorv("Redshift delivery into table {0} failed for stream {1} and the retry window of {2}s is over; "
+ + "the batch stays in s3://{3}/{4}: {5}", table, streamKey, window, pending.bucket, pending.key,
+ describe(e));
+ return true;
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java
index b21dd10bf6..ee08e0b71a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java
@@ -17,7 +17,9 @@ import org.mockito.InOrder;
import org.mockito.Mockito;
import java.nio.charset.StandardCharsets;
+import java.time.Duration;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executor;
@@ -26,6 +28,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
@@ -104,12 +107,19 @@ class FirehoseRedshiftServiceTest {
anyMap(), any());
}
+ /** AWS retries a failed Redshift load every 5 minutes, so 301 seconds is always past it. */
+ private void pastBackoff() {
+ clock.advance(Duration.ofSeconds(301));
+ }
+
@Test
void successfulCopyConsumesTheBatchOnce() throws Exception {
createStream("");
service.flush(STREAM);
service.flush(STREAM);
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
verify(delivery, times(1)).copy(eq("000000000000"), eq("us-east-1"), any(), eq("stage"), anyString());
verifyStagedTimes(1);
@@ -276,4 +286,234 @@ class FirehoseRedshiftServiceTest {
inOrder.verify(mockExecutor).awaitTermination(5, TimeUnit.SECONDS);
inOrder.verify(mockExecutor).shutdownNow();
}
+
+ @Test
+ void failedCopyIsRetriedAgainstTheSameStagedObjectWithoutRestaging() throws Exception {
+ createStream("");
+ copyFails();
+
+ service.flush(STREAM);
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
+
+ ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);
+ verify(delivery, times(3)).copy(anyString(), anyString(), any(), eq("stage"), keys.capture());
+ assertEquals(1, new HashSet<>(keys.getAllValues()).size(), "every attempt must use the one staged key");
+ verifyStagedTimes(1);
+ }
+
+ @Test
+ void failedCopyIsNotRetriedBeforeTheBackoffHasElapsed() throws Exception {
+ createStream("");
+ copyFails();
+
+ service.flush(STREAM);
+ service.retryPendingRedshiftCopies();
+ service.retryPendingRedshiftCopies();
+
+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void theSchedulerTickRetriesPendingCopies() throws Exception {
+ createStream("");
+ copyFails();
+ service.flush(STREAM);
+ pastBackoff();
+
+ service.tickSafely();
+
+ verify(delivery, times(2)).copy(anyString(), anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void recoveredCopyStopsBeingRetried() throws Exception {
+ createStream("");
+ copyFails();
+ service.flush(STREAM);
+ Mockito.reset(delivery);
+
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
+
+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void failedCopyIsDroppedAfterTheRetryWindowButTheStagedObjectStays() throws Exception {
+ createStream(", \"RetryOptions\": {\"DurationInSeconds\": 350}");
+ copyFails();
+
+ service.flush(STREAM);
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
+
+ verify(delivery, times(2)).copy(anyString(), anyString(), any(), anyString(), anyString());
+ verifyStagedTimes(1);
+ }
+
+ @Test
+ void retryWindowShorterThanBackoffExpiresWithoutRetrying() throws Exception {
+ createStream(", \"RetryOptions\": {\"DurationInSeconds\": 10}");
+ copyFails();
+
+ service.flush(STREAM);
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
+
+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());
+ verifyStagedTimes(1);
+ }
+
+ @Test
+ void slowFailureMeasuresNextAttemptFromFailureTime() throws Exception {
+ createStream("");
+ Mockito.doAnswer(invocation -> {
+ clock.advance(Duration.ofSeconds(150));
+ throw new IllegalStateException("slow connection failure");
+ }).when(delivery).copy(anyString(), anyString(), any(), anyString(), anyString());
+
+ service.flush(STREAM);
+ clock.advance(Duration.ofSeconds(151));
+ service.retryPendingRedshiftCopies();
+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());
+
+ clock.advance(Duration.ofSeconds(150));
+ service.retryPendingRedshiftCopies();
+ verify(delivery, times(2)).copy(anyString(), anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void firstAttemptTakingLongerThanRetryWindowDoesNotRetry() throws Exception {
+ createStream(", \"RetryOptions\": {\"DurationInSeconds\": 100}");
+ Mockito.doAnswer(invocation -> {
+ clock.advance(Duration.ofSeconds(150));
+ throw new IllegalStateException("long initial attempt failed");
+ }).when(delivery).copy(anyString(), anyString(), any(), anyString(), anyString());
+
+ service.flush(STREAM);
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
+
+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());
+ verifyStagedTimes(1);
+ }
+
+ @Test
+ void zeroRetryDurationMeansASingleAttempt() throws Exception {
+ createStream(", \"RetryOptions\": {\"DurationInSeconds\": 0}");
+ copyFails();
+
+ service.flush(STREAM);
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
+
+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void anUnexpectedFailureDoesNotEscapeAndIsRetried() throws Exception {
+ createStream("");
+ doThrow(new NullPointerException("unexpected internal bug")).when(delivery)
+ .copy(anyString(), anyString(), any(), anyString(), anyString());
+
+ service.flush(STREAM);
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
+
+ verify(delivery, times(2)).copy(anyString(), anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void aNewerBatchWaitsBehindAnOlderFailingOne() throws Exception {
+ createStream("");
+ copyFails();
+ service.flush(STREAM);
+ service.putRecord(STREAM, new Record("{\"id\":2}".getBytes(StandardCharsets.UTF_8)));
+
+ service.flush(STREAM);
+
+ ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);
+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), eq("stage"), keys.capture());
+ verifyStagedTimes(2);
+ }
+
+ @Test
+ void deletingTheStreamDiscardsPendingCopies() throws Exception {
+ createStream("");
+ copyFails();
+ service.flush(STREAM);
+
+ service.deleteDeliveryStream(STREAM);
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
+
+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void deletingStreamWhileDrainIteratesStopsFurtherCopies() throws Exception {
+ createStream("");
+ copyFails();
+ service.flush(STREAM);
+ service.putRecord(STREAM, new Record("{\"id\":2}".getBytes(StandardCharsets.UTF_8)));
+ service.flush(STREAM);
+
+ Mockito.reset(delivery);
+ Mockito.doAnswer(invocation -> {
+ service.deleteDeliveryStream(STREAM);
+ return null;
+ }).when(delivery).copy(anyString(), anyString(), any(), anyString(), anyString());
+
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
+
+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void retryPendingRedshiftCopiesDoesNotRunWhenHeadIsNotDue() throws Exception {
+ createStream("");
+ copyFails();
+
+ service.flush(STREAM);
+ for (int i = 0; i < 5; i++) {
+ service.retryPendingRedshiftCopies();
+ }
+
+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void pendingCopiesAreDiscardedIfDestinationIsNoLongerRedshift() throws Exception {
+ createStream("");
+ copyFails();
+ service.flush(STREAM);
+
+ service.describeDeliveryStream(STREAM).getDestinations().get(0).setRedshiftDestinationDescription(null);
+
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
+
+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void backupEnabledWritesTheBatchToTheBackupBucketOnlyOnce() throws Exception {
+ createStream(BACKUP_ENABLED);
+ copyFails();
+
+ service.flush(STREAM);
+ pastBackoff();
+ service.retryPendingRedshiftCopies();
+
+ verifyStagedTimes(1);
+ verify(s3Service, times(1)).putObject(eq("bk"), anyString(), any(byte[].class), anyString(), anyMap(), any());
+ }
}