Light, fluffy, and always free - The AWS Local Emulator alternative
fix(firehose): keep buffered records when S3 delivery fails (#3471)
Niels Kootstra6507f32aeaef
diff --git a/docs/services/firehose.md b/docs/services/firehose.mdindex 500e59f590..77e7d2874a 100644--- a/docs/services/firehose.md+++ b/docs/services/firehose.md@@ -65,7 +65,7 @@ Each buffered record is parsed and type-checked against the Glue table's columnsRecords that cannot be converted fail **individually**, as on AWS: the rest of the batch still converts. They are delivered instead to the evaluated `ErrorOutputPrefix`, where `!{firehose:error-output-type}` resolves to `format-conversion-failed`, as one extensionless object holding an NDJSON line per record: `attemptsMade`, `arrivalTimestamp`, `lastErrorCode`, `lastErrorMessage`, `attemptEndingTimestamp`, base64 `rawData`, and a `dataCatalogTable` block. The code is `DataFormatConversion.ParseError` for a record that is not valid JSON, `DataFormatConversion.MalformedData` for one whose values do not fit the column types.-A failure of the batch as a whole is routed rather than discarded: a missing Glue table, a schema Floci cannot convert, or a sidecar failure sends every buffered record to the error output, under `DataFormatConversion.EntityNotFoundException`, `.UnsupportedSchema` and `.InternalError` respectively; those three codes are Floci's own. That routing is only as good as the write it depends on. Floci does not retry a delivery, and a flush whose S3 write fails loses its batch on a DirectPut stream, with or without conversion. The two destination writes of a mixed batch are not atomic either: the error output is written first, so its failure delivers nothing at all, but the reverse remains possible, where the error object lands and the Parquet write then fails, reporting the failed records while the converted ones are gone.+A failure of the batch as a whole is routed rather than discarded: a missing Glue table, a schema Floci cannot convert, or a sidecar failure sends every buffered record to the error output, under `DataFormatConversion.EntityNotFoundException`, `.UnsupportedSchema` and `.InternalError` respectively; those three codes are Floci's own. That routing is only as good as the write it depends on. A flush whose S3 write fails keeps its batch buffered, with or without conversion, and retries it on the next flush ahead of anything buffered since. The two destination writes of a mixed batch are not atomic either: the error output is written first, so its failure delivers nothing at all, but the reverse remains possible, where the error object lands and the Parquet write then fails; the retry then delivers the converted records and reports the failed ones a second time.Validation matches AWS on both `CreateDeliveryStream` and `UpdateDestination`, down to the [modeled](https://docs.aws.amazon.com/firehose/latest/APIReference/API_DataFormatConversionConfiguration.html) bounds, enums and patterns, so a raw JSON client cannot store a configuration AWS would reject. Two rules are worth knowing because they constrain the destination rather than the conversion block: `CompressionFormat` must be `UNCOMPRESSED`, compression being chosen by the output serializer instead, and an explicitly specified `BufferingHints.SizeInMBs` must be at least 64. Omitting `BufferingHints` stores 128 MiB, AWS's larger default for converting streams, rather than the ordinary 5 MiB.diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverter.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverter.javaindex 905a8f0872..e476813a76 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverter.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseParquetConverter.java@@ -204,9 +204,8 @@ public class FirehoseParquetConverter {}// Still before the Parquet: a failure here delivers nothing at all and the- // flush fails as a whole. Nothing is kept for a retry, the staged object goes- // on the way out either way, so recovery belongs to the source: a Kinesis- // stream re-reads its uncommitted checkpoint, a DirectPut buffer cannot.+ // flush fails as a whole. The staged object goes on the way out either way;+ // the batch stays buffered in the service and is retried on the next flush.String errorKey = failures.isEmpty()? null: writeErrorOutput(stream, s3, bucket, failures, deliveryTime, schemaConfig);diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.javaindex 86990d586c..48d84c79f7 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java@@ -66,6 +66,13 @@ public class FirehoseService implements ResourceProvider {private final AccountAwareStorageBackend<DeliveryStreamDescription> streamStore;private final Map<String, List<byte[]>> buffers = new ConcurrentHashMap<>();private final Map<String, Instant> bufferSince = new ConcurrentHashMap<>();+ // One delivery at a time per stream, from taking the buffer to committing or+ // restoring its checkpoint. The buffer monitor alone only makes the snapshot atomic:+ // a second flush could still take records polled past a first flush's checkpoint,+ // deliver them and commit that position while the first write is still in flight.+ // If the first write then failed, the durable checkpoint would already sit past its+ // records, and a restart before the retry would skip them for good.+ private final Map<String, Object> flushLocks = new ConcurrentHashMap<>();private final S3Service s3Service;private final KinesisService kinesisService;private final RegionResolver regionResolver;@@ -752,8 +759,8 @@ public class FirehoseService implements ResourceProvider {}// Rebased on whatever is current rather than on a map carried across the shard loop,- // so that a flush which discarded pending (because its S3 write failed) rolls the- // other shards back to committed instead of having this poll re-assert them.+ // so that a flush which took pending (and, if its S3 write failed, restored it) is+ // never overwritten by a stale copy this poll carried past that flush.private void advanceSourceIterator(String streamKey, String shardId, String iterator) {Map<String, String> advanced = new LinkedHashMap<>(sourceIteratorsInUse(streamKey));advanced.put(shardId, iterator);@@ -848,35 +855,38 @@ public class FirehoseService implements ResourceProvider {return;}- List<byte[]> toFlush;- // Taken with the records, under the same lock: this is the position the records- // about to be written cover, and committing it is this flush's job.- Map<String, String> checkpoint;- synchronized (buffer) {- toFlush = new ArrayList<>(buffer);- buffer.clear();- bufferSince.remove(streamName);- checkpoint = pendingSourceIterators.remove(streamName);- }- if (toFlush.isEmpty()) {- // Lost the race against a concurrent flush; nothing left to deliver. Any- // checkpoint taken here covers no undelivered records -- it can only have- // come from a poll that read an empty page -- so it commits as it stands.- commitSourceIterators(streamName, checkpoint);- return;- }+ synchronized (flushLocks.computeIfAbsent(streamName, k -> new Object())) {+ List<byte[]> toFlush;+ // Taken with the records, under the same lock: this is the position the records+ // about to be written cover, and committing it is this flush's job.+ Map<String, String> checkpoint;+ Instant since;+ synchronized (buffer) {+ toFlush = new ArrayList<>(buffer);+ buffer.clear();+ since = bufferSince.remove(streamName);+ checkpoint = pendingSourceIterators.remove(streamName);+ }+ if (toFlush.isEmpty()) {+ // Lost the race for the stream lock; nothing left to deliver. Any+ // checkpoint taken here covers no undelivered records -- it can only have+ // come from a poll that read an empty page -- so it commits as it stands.+ commitSourceIterators(streamName, checkpoint);+ return;+ }- // Every store this delivery touches, Glue and S3 included, reads the account- // from the request context, and a scheduled flush has none. Without this the- // work would run as the default account rather than the stream's owner: the- // stream's Glue table would be missed and its objects would land in the wrong- // partition. The sidecar follows the same context, so both sides stay together.- RequestScopes.runAs(stream.getAccountId(),- () -> deliverBuffer(streamName, stream, toFlush, checkpoint));+ // Every store this delivery touches, Glue and S3 included, reads the account+ // from the request context, and a scheduled flush has none. Without this the+ // work would run as the default account rather than the stream's owner: the+ // stream's Glue table would be missed and its objects would land in the wrong+ // partition. The sidecar follows the same context, so both sides stay together.+ RequestScopes.runAs(stream.getAccountId(),+ () -> deliverBuffer(streamName, stream, toFlush, since, checkpoint));+ }}private void deliverBuffer(String streamName, DeliveryStreamDescription stream,- List<byte[]> toFlush, Map<String, String> checkpoint) {+ List<byte[]> toFlush, Instant since, Map<String, String> checkpoint) {try {String bucket = resolveBucket(stream);S3Destination s3 = stream.s3Destination();@@ -935,10 +945,34 @@ public class FirehoseService implements ResourceProvider {// Only now: the records these iterators were read past are durable.commitSourceIterators(streamName, checkpoint);} catch (Exception e) {- LOG.errorv("Failed to flush Firehose stream {0}: {1}", streamName, e.getMessage());- // checkpoint is deliberately neither committed nor put back. The durable- // checkpoint stays where it was, so the next poll reads these records again- // and this failed delivery repairs itself.+ LOG.errorv("Failed to flush Firehose stream {0}; keeping {1} records buffered for retry: {2}",+ streamName, toFlush.size(), e.getMessage());+ restoreFailedFlush(streamName, toFlush, since, checkpoint);+ }+ }++ private void restoreFailedFlush(String streamName, List<byte[]> toFlush, Instant since,+ Map<String, String> checkpoint) {+ List<byte[]> buffer = buffers.get(streamName);+ if (buffer != null) {+ synchronized (buffer) {+ List<byte[]> restored = new ArrayList<>(toFlush);+ restored.addAll(buffer);+ buffer.clear();+ buffer.addAll(restored);+ // The restored batch is the oldest data in the buffer again, so the buffering+ // window starts where it did before the failed flush, not at a record that+ // arrived while the write was in flight.+ bufferSince.put(streamName, since != null ? since : clock.instant());+ if (checkpoint != null) {+ Map<String, String> pending = new LinkedHashMap<>(checkpoint);+ Map<String, String> concurrent = pendingSourceIterators.get(streamName);+ if (concurrent != null) {+ pending.putAll(concurrent);+ }+ pendingSourceIterators.put(streamName, Collections.unmodifiableMap(pending));+ }+ }}}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.javaindex 97f895c3d5..52eec55d0f 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java@@ -21,11 +21,22 @@ import org.mockito.Mockito;import java.io.IOException;import java.nio.charset.StandardCharsets;+import java.time.Duration;import java.time.Instant;import java.util.Arrays;import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.Optional;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.concurrent.atomic.AtomicReference;+import java.util.function.UnaryOperator;import static org.junit.jupiter.api.Assertions.assertArrayEquals;import static org.junit.jupiter.api.Assertions.assertEquals;@@ -38,6 +49,7 @@ import static org.mockito.ArgumentMatchers.anyMap;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.argThat;import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;import static org.mockito.Mockito.times;@@ -77,6 +89,14 @@ class FirehoseServiceTest {}private FirehoseService newService(int flushRecordCount) {+ return newService(flushRecordCount, UnaryOperator.identity());+ }++ /**+ * @param kinesisDecorator applied to the source service before the poller sees it,+ * e.g. {@code Mockito::spy} to park a poll mid-way.+ */+ private FirehoseService newService(int flushRecordCount, UnaryOperator<KinesisService> kinesisDecorator) {EmulatorConfig.FirehoseServiceConfig firehoseCfg = mock(EmulatorConfig.FirehoseServiceConfig.class);when(firehoseCfg.enabled()).thenReturn(true);when(firehoseCfg.tickIntervalSeconds()).thenReturn(10L);@@ -89,12 +109,37 @@ class FirehoseServiceTest {RegionResolver regionResolver = new RegionResolver("us-east-1", "000000000000");// A real KinesisService, not a mock: the point of the source poller is that it// sees what a GetRecords consumer sees, and a stubbed one could not show that.- kinesisService = new KinesisService(storageFactory, regionResolver);+ kinesisService = kinesisDecorator.apply(new KinesisService(storageFactory, regionResolver));return new FirehoseService(storageFactory, s3Service, kinesisService,regionResolver, clock, config, mock(FirehoseParquetConverter.class),mock(FirehoseLambdaTransformer.class));}+ /** The committed source checkpoint, as a restarted service would read it back. */+ @SuppressWarnings("unchecked")+ private Optional<Map<String, String>> durableCheckpoint(String deliveryStream) {+ AccountAwareStorageBackend<Map<String, String>> store =+ (AccountAwareStorageBackend<Map<String, String>>) backends.get("firehose/source-iterators.json");+ return store.getForAccount("000000000000", "us-east-1/" + deliveryStream);+ }++ /**+ * Spin until {@code thread} is parked entering a monitor or {@code task} has finished,+ * whichever comes first, so the assertions that follow see a settled state rather than+ * a thread that was merely never scheduled.+ */+ private static void awaitParkedOrDone(AtomicReference<Thread> thread, Future<?> task)+ throws InterruptedException {+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);+ while (System.nanoTime() < deadline && !task.isDone()) {+ Thread th = thread.get();+ if (th != null && th.getState() == Thread.State.BLOCKED) {+ return;+ }+ Thread.sleep(5);+ }+ }+private static S3Destination destination(String bucketArn, String compressionFormat) {S3Destination s3 = new S3Destination();s3.setBucketArn(bucketArn);@@ -125,13 +170,18 @@ class FirehoseServiceTest {}private Delivered delivered(String expectedBucket) {+ return delivered(expectedBucket, 1);+ }++ /** The most recent of exactly {@code attempts} S3 writes. */+ private Delivered delivered(String expectedBucket, int attempts) {ArgumentCaptor<String> bucket = ArgumentCaptor.forClass(String.class);ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);ArgumentCaptor<String> contentType = ArgumentCaptor.forClass(String.class);ArgumentCaptor<PutObjectOptions> options = ArgumentCaptor.forClass(PutObjectOptions.class);- verify(s3Service).putObject(bucket.capture(), key.capture(), body.capture(), contentType.capture(),- anyMap(), options.capture());+ verify(s3Service, times(attempts)).putObject(bucket.capture(), key.capture(), body.capture(),+ contentType.capture(), anyMap(), options.capture());assertEquals(expectedBucket, bucket.getValue());return new Delivered(key.getValue(), body.getValue(), contentType.getValue(),options.getValue().getContentEncoding());@@ -673,6 +723,154 @@ class FirehoseServiceTest {assertEquals("hello\n", delivered("sink").text());}+ @Test+ void aFailedDeliveryRestoresTheBatchAheadOfRecordsAddedDuringDelivery() {+ firehoseService.createDeliveryStream("retry-stream", destination("arn:aws:s3:::sink", null));+ firehoseService.putRecord("retry-stream", new Record("first".getBytes(StandardCharsets.UTF_8)));+ firehoseService.putRecord("retry-stream", new Record("second".getBytes(StandardCharsets.UTF_8)));+ AtomicBoolean s3Down = new AtomicBoolean(true);+ when(s3Service.putObject(anyString(), anyString(), any(byte[].class), anyString(),+ anyMap(), any(PutObjectOptions.class))).thenAnswer(invocation -> {+ if (s3Down.getAndSet(false)) {+ firehoseService.putRecord("retry-stream", new Record("third".getBytes(StandardCharsets.UTF_8)));+ throw new RuntimeException("s3 unavailable");+ }+ return null;+ });++ firehoseService.flush("retry-stream");+ firehoseService.flush("retry-stream");++ assertEquals("first\nsecond\nthird\n", delivered("sink", 2).text());+ }++ @Test+ void aRestoredBatchIsRetriedOnTheNextScheduledFlush() {+ firehoseService.createDeliveryStream("retry-stream", destination("arn:aws:s3:::sink", null));+ firehoseService.putRecord("retry-stream", new Record("first".getBytes(StandardCharsets.UTF_8)));+ AtomicBoolean s3Down = new AtomicBoolean(true);+ when(s3Service.putObject(anyString(), anyString(), any(byte[].class), anyString(),+ anyMap(), any(PutObjectOptions.class))).thenAnswer(invocation -> {+ if (s3Down.getAndSet(false)) {+ // A record arrives while the write is in flight; it must not push the+ // restored batch's buffering start forward.+ clock.advance(Duration.ofSeconds(1));+ firehoseService.putRecord("retry-stream", new Record("second".getBytes(StandardCharsets.UTF_8)));+ throw new RuntimeException("s3 unavailable");+ }+ return null;+ });+ clock.advance(Duration.ofSeconds(300));+ firehoseService.flushDueBuffers(clock.instant());+ verify(s3Service, times(1)).putObject(anyString(), anyString(), any(byte[].class), anyString(),+ anyMap(), any(PutObjectOptions.class));++ clock.advance(Duration.ofSeconds(10));+ firehoseService.flushDueBuffers(clock.instant());++ assertEquals("first\nsecond\n", delivered("sink", 2).text());+ }++ @Test+ void aSuccessfulDeliveryClearsOnlyTheRecordsItWrote() {+ firehoseService.createDeliveryStream("keep-stream", destination("arn:aws:s3:::sink", null));+ firehoseService.putRecord("keep-stream", new Record("first".getBytes(StandardCharsets.UTF_8)));+ AtomicBoolean firstWrite = new AtomicBoolean(true);+ when(s3Service.putObject(anyString(), anyString(), any(byte[].class), anyString(),+ anyMap(), any(PutObjectOptions.class))).thenAnswer(invocation -> {+ if (firstWrite.getAndSet(false)) {+ firehoseService.putRecord("keep-stream", new Record("second".getBytes(StandardCharsets.UTF_8)));+ }+ return null;+ });++ firehoseService.flush("keep-stream");+ assertEquals("first\n", delivered("sink").text());++ firehoseService.flush("keep-stream");+ assertEquals("second\n", delivered("sink", 2).text());+ }++ @Test+ void aLaterFlushCannotCommitItsCheckpointWhileAnEarlierBatchIsUnresolved() throws Exception {+ // A spy, so one poll can be parked between reading its page and buffering it.+ firehoseService = newService(0, Mockito::spy);+ createSourcedDeliveryStream("sourced-stream", "src-stream");+ kinesisService.putRecord("src-stream", "first".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");+ firehoseService.pollKinesisSources();++ // The first S3 write parks until released and then fails, like a slow S3 that+ // eventually answers with an error. Later writes succeed.+ CountDownLatch firstWriteStarted = new CountDownLatch(1);+ CountDownLatch releaseFirstWrite = new CountDownLatch(1);+ AtomicBoolean firstWrite = new AtomicBoolean(true);+ when(s3Service.putObject(anyString(), anyString(), any(byte[].class), anyString(),+ anyMap(), any(PutObjectOptions.class))).thenAnswer(invocation -> {+ if (firstWrite.getAndSet(false)) {+ firstWriteStarted.countDown();+ releaseFirstWrite.await();+ throw new RuntimeException("s3 unavailable");+ }+ return null;+ });+ // The next poll parks once it has read its page, before it buffers the records+ // and advances the pending checkpoint past them.+ kinesisService.putRecord("src-stream", "second".getBytes(StandardCharsets.UTF_8), "pk", "us-east-1");+ CountDownLatch pollRead = new CountDownLatch(1);+ CountDownLatch resumePoll = new CountDownLatch(1);+ doAnswer(invocation -> {+ Object page = invocation.callRealMethod();+ pollRead.countDown();+ resumePoll.await();+ return page;+ }).when(kinesisService).getRecordsForAccount(anyString(), anyString(), any(), anyString());++ ExecutorService pool = Executors.newFixedThreadPool(3);+ try {+ // P reads "second" from the position "first" was polled to, then parks.+ Future<?> poll = pool.submit(firehoseService::pollKinesisSources);+ assertTrue(pollRead.await(2, TimeUnit.SECONDS), "the poll must read its page");+ // A takes "first" with its checkpoint and parks inside the S3 write.+ Future<?> flushA = pool.submit(() -> firehoseService.flush("sourced-stream"));+ assertTrue(firstWriteStarted.await(2, TimeUnit.SECONDS), "flush A must reach its S3 write");+ // P buffers "second" and leaves a checkpoint past both records pending.+ resumePoll.countDown();+ poll.get(2, TimeUnit.SECONDS);+ // B finds "second" buffered while A still owes "first" to S3.+ AtomicReference<Thread> bThread = new AtomicReference<>();+ Future<?> flushB = pool.submit(() -> {+ bThread.set(Thread.currentThread());+ firehoseService.flush("sourced-stream");+ });+ awaitParkedOrDone(bThread, flushB);++ // Committing B's checkpoint now would persist a position past "first" while+ // "first" is not durable: a restart before A's retry would skip it forever.+ assertTrue(durableCheckpoint("sourced-stream").isEmpty(),+ "no checkpoint may be committed while an earlier batch is unresolved");+ verify(s3Service, times(1)).putObject(anyString(), anyString(), any(byte[].class), anyString(),+ anyMap(), any(PutObjectOptions.class));+ assertThrows(TimeoutException.class, () -> flushB.get(500, TimeUnit.MILLISECONDS),+ "a later flush must wait for the earlier batch to resolve");++ releaseFirstWrite.countDown();+ flushA.get(2, TimeUnit.SECONDS);+ flushB.get(2, TimeUnit.SECONDS);+ } finally {+ pool.shutdownNow();+ }+ // A's failure hands "first" back ahead of "second"; B then delivers both.+ assertEquals("first\nsecond\n", delivered("sink", 2).text());++ // Restart: the committed checkpoint covers exactly what reached S3, so nothing+ // is re-read and nothing is skipped.+ firehoseService = newService(0);+ firehoseService.pollKinesisSources();+ firehoseService.flush("sourced-stream");+ verify(s3Service, times(2)).putObject(anyString(), anyString(), any(byte[].class), anyString(),+ anyMap(), any(PutObjectOptions.class));+ }+@Testvoid aMissingSourceStreamDoesNotStopOtherStreamsFromPolling() {// No createStream for "gone-stream": describeStream throws ResourceNotFound.