Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(firehose): apply the Lambda transform to buffered records (#3393)

Kenshin Okinakad96516dc70eb
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseLambdaTransformDeliveryTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseLambdaTransformDeliveryTest.java
new file mode 100644
index 0000000000..f76ce0756c
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseLambdaTransformDeliveryTest.java
@@ -0,0 +1,251 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+import software.amazon.awssdk.core.SdkBytes;
+import software.amazon.awssdk.services.firehose.FirehoseClient;
+import software.amazon.awssdk.services.firehose.model.BufferingHints;
+import software.amazon.awssdk.services.firehose.model.CompressionFormat;
+import software.amazon.awssdk.services.firehose.model.CreateDeliveryStreamRequest;
+import software.amazon.awssdk.services.firehose.model.DeleteDeliveryStreamRequest;
+import software.amazon.awssdk.services.firehose.model.DeliveryStreamStatus;
+import software.amazon.awssdk.services.firehose.model.DeliveryStreamType;
+import software.amazon.awssdk.services.firehose.model.DescribeDeliveryStreamRequest;
+import software.amazon.awssdk.services.firehose.model.ExtendedS3DestinationConfiguration;
+import software.amazon.awssdk.services.firehose.model.ProcessingConfiguration;
+import software.amazon.awssdk.services.firehose.model.Processor;
+import software.amazon.awssdk.services.firehose.model.ProcessorParameter;
+import software.amazon.awssdk.services.firehose.model.ProcessorParameterName;
+import software.amazon.awssdk.services.firehose.model.ProcessorType;
+import software.amazon.awssdk.services.firehose.model.PutRecordBatchRequest;
+import software.amazon.awssdk.services.firehose.model.Record;
+import software.amazon.awssdk.services.lambda.LambdaClient;
+import software.amazon.awssdk.services.lambda.model.CreateFunctionRequest;
+import software.amazon.awssdk.services.lambda.model.DeleteFunctionRequest;
+import software.amazon.awssdk.services.lambda.model.FunctionCode;
+import software.amazon.awssdk.services.lambda.model.Runtime;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.List;
+import java.util.Optional;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * End-to-end proof that a transforming stream really invokes its function: the
+ * records put through the SDK come back out of S3 uppercased by a Node.js
+ * handler, the one it dropped is nowhere, and the one it failed lands under the
+ * evaluated ErrorOutputPrefix as an AWS-shaped NDJSON error line.
+ *
+ * This is the only layer that runs the function for real. The emulator's own
+ * tests mock the Lambda service, so the invocation payload, the base64 round
+ * trip through the runtime, and the ARN resolution from the flusher thread are
+ * exercised here and nowhere else.
+ */
+@DisplayName("Firehose record transformation delivery (Lambda)")
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class FirehoseLambdaTransformDeliveryTest {
+
+ private static final int BUFFERING_INTERVAL_SECONDS = 5;
+ private static final long POLL_TIMEOUT_MILLIS = 240_000L;
+ private static final String KEPT_RECORD = "keep-me\n";
+ private static final String DROPPED_RECORD = "DROP-me\n";
+ private static final String FAILED_RECORD = "FAIL-me\n";
+
+ private static FirehoseClient firehose;
+ private static LambdaClient lambda;
+ private static S3Client s3;
+ private static String bucket;
+ private static String streamName;
+ private static String functionName;
+
+ @BeforeAll
+ static void setup() throws InterruptedException {
+ String firehoseRole = TestFixtures.isRealAws()
+ ? System.getenv("FIREHOSE_ROLE_ARN")
+ : "arn:aws:iam::000000000000:role/firehose-role";
+ String lambdaRole = TestFixtures.isRealAws()
+ ? System.getenv("LAMBDA_ROLE_ARN")
+ : "arn:aws:iam::000000000000:role/lambda-role";
+ if (TestFixtures.isRealAws() && (firehoseRole == null || lambdaRole == null)) {
+ Assumptions.abort("FIREHOSE_ROLE_ARN and LAMBDA_ROLE_ARN not set");
+ }
+
+ firehose = TestFixtures.firehoseClient();
+ lambda = TestFixtures.lambdaClient();
+ s3 = TestFixtures.s3Client();
+ bucket = TestFixtures.uniqueName("firehose-transform");
+ streamName = TestFixtures.uniqueName("sdk-transform");
+ functionName = TestFixtures.uniqueName("sdk-transform-fn");
+
+ s3.createBucket(CreateBucketRequest.builder().bucket(bucket).build());
+ String functionArn = lambda.createFunction(CreateFunctionRequest.builder()
+ .functionName(functionName)
+ .runtime(Runtime.NODEJS20_X)
+ .role(lambdaRole)
+ .handler("index.handler")
+ .timeout(30)
+ .memorySize(256)
+ .code(FunctionCode.builder()
+ .zipFile(SdkBytes.fromByteArray(LambdaUtils.firehoseTransformZip()))
+ .build())
+ .build()).functionArn();
+
+ firehose.createDeliveryStream(CreateDeliveryStreamRequest.builder()
+ .deliveryStreamName(streamName)
+ .deliveryStreamType(DeliveryStreamType.DIRECT_PUT)
+ .extendedS3DestinationConfiguration(ExtendedS3DestinationConfiguration.builder()
+ .roleARN(firehoseRole)
+ .bucketARN("arn:aws:s3:::" + bucket)
+ .prefix("data/")
+ .errorOutputPrefix("errors/!{firehose:error-output-type}/")
+ .compressionFormat(CompressionFormat.UNCOMPRESSED)
+ .bufferingHints(BufferingHints.builder()
+ .sizeInMBs(5)
+ .intervalInSeconds(BUFFERING_INTERVAL_SECONDS)
+ .build())
+ .processingConfiguration(ProcessingConfiguration.builder()
+ .enabled(true)
+ .processors(Processor.builder()
+ .type(ProcessorType.LAMBDA)
+ .parameters(ProcessorParameter.builder()
+ .parameterName(ProcessorParameterName.LAMBDA_ARN)
+ .parameterValue(functionArn)
+ .build())
+ .build())
+ .build())
+ .build())
+ .build());
+
+ awaitStreamActive();
+
+ assertThat(firehose.putRecordBatch(PutRecordBatchRequest.builder()
+ .deliveryStreamName(streamName)
+ .records(record(KEPT_RECORD), record(DROPPED_RECORD), record(FAILED_RECORD))
+ .build()).failedPutCount()).isZero();
+ }
+
+ @Test
+ @Order(1)
+ @DisplayName("the delivered object holds what the function returned, and nothing it dropped")
+ void deliversTheTransformedRecord() throws InterruptedException {
+ String delivered = new String(awaitObject("data/"), StandardCharsets.UTF_8);
+
+ assertThat(delivered).contains("KEEP-ME");
+ assertThat(delivered).doesNotContain("keep-me");
+ assertThat(delivered).doesNotContain("DROP-me").doesNotContain("DROP-ME");
+ assertThat(delivered).doesNotContain("FAIL-me").doesNotContain("FAIL-ME");
+ }
+
+ @Test
+ @Order(2)
+ @DisplayName("a ProcessingFailed record lands under the processing-failed error output")
+ void writesTheFailedRecordToTheErrorOutput() throws InterruptedException {
+ String errorLine = new String(awaitObject("errors/processing-failed/"), StandardCharsets.UTF_8).strip();
+
+ assertThat(errorLine).contains("\"errorCode\":\"Lambda.ProcessingFailedStatus\"");
+ assertThat(errorLine).contains("\"lambdaARN\":\"");
+ assertThat(errorLine).contains("\"rawData\":\""
+ + Base64.getEncoder().encodeToString(FAILED_RECORD.getBytes(StandardCharsets.UTF_8)) + "\"");
+ }
+
+ /**
+ * The buffer has to flush and the function has to run, which on a cold container is
+ * well past the buffering interval, so both tests poll rather than sleep once.
+ */
+ private static byte[] awaitObject(String prefix) throws InterruptedException {
+ long deadline = System.currentTimeMillis() + POLL_TIMEOUT_MILLIS;
+ while (true) {
+ Optional<S3Object> found = firstObject(prefix);
+ if (found.isPresent()) {
+ return s3.getObjectAsBytes(GetObjectRequest.builder()
+ .bucket(bucket).key(found.get().key()).build()).asByteArray();
+ }
+ if (System.currentTimeMillis() >= deadline) {
+ throw new AssertionError("no object under " + prefix + " in " + bucket + " after "
+ + POLL_TIMEOUT_MILLIS / 1000 + "s");
+ }
+ Thread.sleep(5_000L);
+ }
+ }
+
+ private static Optional<S3Object> firstObject(String prefix) {
+ List<S3Object> objects = s3.listObjectsV2(ListObjectsV2Request.builder()
+ .bucket(bucket).prefix(prefix).build()).contents();
+ return objects.isEmpty() ? Optional.empty() : Optional.of(objects.get(0));
+ }
+
+ // Real AWS returns from CreateDeliveryStream while the stream is still CREATING
+ // and rejects PutRecordBatch until it turns ACTIVE; Floci creates it active, so
+ // this returns on the first poll locally.
+ private static void awaitStreamActive() throws InterruptedException {
+ long deadline = System.currentTimeMillis() + POLL_TIMEOUT_MILLIS;
+ while (true) {
+ DeliveryStreamStatus status = firehose.describeDeliveryStream(
+ DescribeDeliveryStreamRequest.builder()
+ .deliveryStreamName(streamName).build())
+ .deliveryStreamDescription().deliveryStreamStatus();
+ if (status == DeliveryStreamStatus.ACTIVE) {
+ return;
+ }
+ if (System.currentTimeMillis() >= deadline) {
+ throw new AssertionError("delivery stream stayed " + status + " for "
+ + POLL_TIMEOUT_MILLIS / 1000 + "s");
+ }
+ Thread.sleep(5_000L);
+ }
+ }
+
+ private static Record record(String data) {
+ return Record.builder().data(SdkBytes.fromString(data, StandardCharsets.UTF_8)).build();
+ }
+
+ @AfterAll
+ static void cleanup() {
+ if (firehose != null) {
+ try {
+ firehose.deleteDeliveryStream(DeleteDeliveryStreamRequest.builder()
+ .deliveryStreamName(streamName).build());
+ } catch (Exception e) {
+ // Best effort: a stream that never got created must not fail the run.
+ System.out.println("Cleanup of delivery stream " + streamName + " failed: " + e);
+ }
+ firehose.close();
+ }
+ if (lambda != null) {
+ try {
+ lambda.deleteFunction(DeleteFunctionRequest.builder()
+ .functionName(functionName).build());
+ } catch (Exception e) {
+ // Best effort cleanup of the transform function.
+ System.out.println("Cleanup of function " + functionName + " failed: " + e);
+ }
+ lambda.close();
+ }
+ if (s3 != null) {
+ try {
+ s3.listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()).contents()
+ .forEach(object -> s3.deleteObject(DeleteObjectRequest.builder()
+ .bucket(bucket).key(object.key()).build()));
+ s3.deleteBucket(builder -> builder.bucket(bucket));
+ } catch (Exception e) {
+ // Best effort cleanup of the probe bucket.
+ System.out.println("Cleanup of bucket " + bucket + " failed: " + e);
+ }
+ s3.close();
+ }
+ }
+}
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java
index d559780c70..fa7de92808 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java
@@ -32,6 +32,33 @@ public final class LambdaUtils {
return createZip("index.js", code);
}
+ /**
+ * ZIP containing a Node.js handler shaped like a Firehose transform: it uppercases each
+ * record, drops one whose payload says DROP, and reports one whose payload says FAIL,
+ * so a single invocation exercises all three result values.
+ */
+ public static byte[] firehoseTransformZip() {
+ String code = """
+ exports.handler = async (event) => ({
+ records: (event.records || []).map((r) => {
+ const data = Buffer.from(r.data, 'base64').toString('utf8');
+ if (data.includes('DROP')) {
+ return { recordId: r.recordId, result: 'Dropped' };
+ }
+ if (data.includes('FAIL')) {
+ return { recordId: r.recordId, result: 'ProcessingFailed' };
+ }
+ return {
+ recordId: r.recordId,
+ result: 'Ok',
+ data: Buffer.from(data.toUpperCase(), 'utf8').toString('base64')
+ };
+ })
+ });
+ """;
+ return createZip("index.js", code);
+ }
+
/**
* ZIP containing a Ruby handler that greets by name.
*/
diff --git a/docs/services/firehose.md b/docs/services/firehose.md
index 1d579f5c7f..500e59f590 100644
--- a/docs/services/firehose.md
+++ b/docs/services/firehose.md
@@ -85,15 +85,19 @@ An omitted `Enabled` counts as enabled; `Enabled: false` stores and merges the c
## Record transformation
-`ProcessingConfiguration` on the extended S3 destination is modelled, validated and echoed, but **not applied**: a stream that enables a transformation is accepted and delivers its records untransformed, and create and update log a warning saying so. Validation follows AWS, down to the messages and the order they are reported in.
+`ProcessingConfiguration` on the extended S3 destination is applied at delivery: a flushed buffer goes to the Lambda processor's function as one invocation, `Ok` records deliver the returned data, and `Dropped` records leave no trace. Any other per-record outcome goes to the error output while the rest of the batch still delivers, one NDJSON line each under the `ErrorOutputPrefix` with `!{firehose:error-output-type}` resolving to `processing-failed`.
-Two behaviors are worth knowing because they differ from the conversion block above. `UpdateDestination` replaces this block whole rather than merging it member-wise, so an update carrying only `{"Enabled": false}` leaves no processors behind, and each update is validated against its own content rather than the merged result. And a stored Lambda processor is echoed with `NumberOfRetries` and `RoleArn` filled in and its parameters reordered, as AWS returns them.
+A function that errors is retried `NumberOfRetries` more times and then fails the whole batch as `Lambda.FunctionError`; a response the function returned successfully is final however malformed it is. The transformation runs before data format conversion, so a stream configured for both converts what the function returned.
+
+Two behaviors invert what the conversion block above does, both probed rather than assumed: `UpdateDestination` replaces this block whole instead of merging it member-wise, so an update carrying only `{"Enabled": false}` leaves no processors behind and is validated against its own content; and an omitted `Enabled` leaves the transformation **disabled**, where on the conversion block it means enabled. A stored Lambda processor is echoed with `NumberOfRetries`, `RoleArn`, `BufferSizeInMBs` and `BufferIntervalInSeconds` filled in and reordered, as AWS returns them; the two buffer values default to 1 and 60, the processor's own defaults rather than anything `BufferingHints` says.
### Known deviations from AWS
-- The transformation is not applied. A transform function is never invoked, so records reach the destination exactly as they were put, which is what makes a local test of one pass where AWS would not.
-- Two checks AWS itself does not make are deliberately absent: `NumberOfRetries` is not range-checked, despite the documented 1 to 8, and the function a `LambdaArn` names is not required to exist.
-- A null entry in `Processors` or `Parameters`, which a raw JSON client can send, is ignored rather than reported. Real AWS answers `InternalFailure` there, a fault of its own that is not worth reproducing; skipping the entry leaves the same configuration as omitting it would.
+- Only the `Lambda` processor type is applied; the others are stored and echoed.
+- The whole buffer is one invocation, so the processor's buffering parameters do not re-buffer and no payload size limit is enforced.
+- `recordId` is a UUID rather than AWS's longer sequence token, and every timestamp is the delivery time: Floci tracks no per-record arrival.
+- Neither `NumberOfRetries` nor the existence of the function a `LambdaArn` names is checked at configuration time, as on AWS. At delivery an ARN naming nothing fails as `Lambda.FunctionError`, and a `NumberOfRetries` above 100 is capped, a bound AWS does not impose so that one flush cannot occupy the flusher indefinitely.
+- A null entry in `Processors` or `Parameters`, which a raw JSON client can send, is ignored rather than reported; real AWS answers `InternalFailure` there, a fault of its own not worth reproducing.
## S3 object keys
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformer.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformer.java
new file mode 100644
index 0000000000..498a84a951
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformer.java
@@ -0,0 +1,388 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsArnUtils;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
+import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.lambda.model.InvocationType;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.io.ByteArrayOutputStream;
+import java.math.BigInteger;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * Runs a flushed batch through the stream's Lambda transform before it is
+ * delivered: one invocation per batch carrying every buffered record, the
+ * returned data replacing the original, and records the function dropped or
+ * failed taken out of the delivery. The failed ones go to the error output,
+ * one NDJSON line each, under the evaluated ErrorOutputPrefix with
+ * {@code !{firehose:error-output-type}} resolved to {@code processing-failed}.
+ *
+ * The per-record semantics were probed against real AWS (us-west-2,
+ * 2026-09-10): {@code Ok} delivers the returned data, {@code Dropped} leaves no
+ * trace at all, and anything else routes that record to the error output while
+ * the rest of the batch still delivers. A function that errors is retried, a
+ * response the function shaped wrongly is not. Every message below is AWS's own
+ * except the two {@code Lambda.InvalidReturnFormat} ones this class invents, for
+ * a {@code result} AWS does not define and for {@code data} that is not base64.
+ *
+ * This runs ahead of data format conversion, so a stream with both configured
+ * converts what the transform returned.
+ */
+@ApplicationScoped
+public class FirehoseLambdaTransformer {
+
+ private static final Logger LOG = Logger.getLogger(FirehoseLambdaTransformer.class);
+ private static final String ERROR_OUTPUT_TYPE = "processing-failed";
+ private static final String CONTENT_TYPE = "application/octet-stream";
+ private static final int DEFAULT_NUMBER_OF_RETRIES = 3;
+ private static final int MAX_NUMBER_OF_RETRIES = 100;
+ private static final String RESULT_OK = "Ok";
+ private static final String RESULT_DROPPED = "Dropped";
+ private static final String RESULT_PROCESSING_FAILED = "ProcessingFailed";
+
+ private static final String FUNCTION_ERROR_MESSAGE =
+ "The Lambda function was successfully invoked but it returned an error result.";
+ private static final String MISSING_RECORD_ID_MESSAGE =
+ "One or more record Ids were not returned. Ensure that the Lambda function returns"
+ + " all received record Ids.";
+ private static final String DUPLICATED_RECORD_ID_MESSAGE =
+ "Multiple records were returned with the same record Id. Ensure that the Lambda"
+ + " function returns a unique record Id for each record.";
+ private static final String PROCESSING_FAILED_MESSAGE = "ProcessingFailed status set for record";
+ private static final String NULL_DATA_MESSAGE =
+ "The data field cannot be null when the SourceType is DirectPut or KinesisStreamAsSource"
+ + " and processing result is Ok. The value field cannot be null when the SourceType"
+ + " is MSKAsSource and processing result is Ok.";
+
+ private final LambdaService lambdaService;
+ private final S3Service s3Service;
+ private final ObjectMapper mapper;
+ private final RegionResolver regionResolver;
+
+ @Inject
+ public FirehoseLambdaTransformer(LambdaService lambdaService, S3Service s3Service,
+ ObjectMapper mapper, RegionResolver regionResolver) {
+ this.lambdaService = lambdaService;
+ this.s3Service = s3Service;
+ this.mapper = mapper;
+ this.regionResolver = regionResolver;
+ }
+
+ /** What a transformation produced: the records still to deliver, plus the counts for the flush log line. */
+ public record Outcome(List<byte[]> records, int droppedRecords, int failedRecords, String errorKey) {}
+
+ public Outcome transform(DeliveryStreamDescription stream, String bucket, List<byte[]> records,
+ Instant deliveryTime) {
+ S3Destination s3 = stream.s3Destination();
+ Processor processor = lambdaProcessor(s3);
+ if (processor == null) {
+ return new Outcome(records, 0, 0, null);
+ }
+ String functionArn = parameterValue(processor, "LambdaArn");
+
+ List<String> recordIds = new ArrayList<>(records.size());
+ for (int i = 0; i < records.size(); i++) {
+ recordIds.add(UUID.randomUUID().toString());
+ }
+
+ Invocation invocation = invoke(stream, functionArn, recordIds, records, deliveryTime,
+ numberOfRetries(processor) + 1);
+
+ List<byte[]> delivered = new ArrayList<>(records.size());
+ List<FailedRecord> failures = new ArrayList<>();
+ int dropped = 0;
+ for (int i = 0; i < records.size(); i++) {
+ byte[] record = records.get(i);
+ Verdict verdict = invocation.verdictFor(recordIds.get(i));
+ if (verdict.errorCode() != null) {
+ failures.add(new FailedRecord(record, verdict.errorCode(), verdict.errorMessage()));
+ } else if (verdict.data() == null) {
+ dropped++;
+ } else {
+ delivered.add(verdict.data());
+ }
+ }
+
+ String errorKey = failures.isEmpty() ? null
+ : writeErrorOutput(stream, s3, bucket, failures, deliveryTime, functionArn,
+ invocation.attemptsMade());
+ return new Outcome(delivered, dropped, failures.size(), errorKey);
+ }
+
+ // ── configuration ────────────────────────────────────────────────────────
+
+ /**
+ * The single Lambda processor of an enabled ProcessingConfiguration, or null when the
+ * stream has none: the other processor types AWS models are stored and echoed but not
+ * applied. The validator has already made "at most one" true.
+ */
+ private static Processor lambdaProcessor(S3Destination s3) {
+ if (s3 == null || !s3.isProcessingEnabled() || s3.getProcessingConfiguration().getProcessors() == null) {
+ return null;
+ }
+ for (Processor processor : s3.getProcessingConfiguration().getProcessors()) {
+ if (processor != null && "Lambda".equals(processor.getType())
+ && parameterValue(processor, "LambdaArn") != null) {
+ return processor;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * AWS does not range-check NumberOfRetries at configuration time, so a stored value
+ * can be any number the parameter's string type allows, 2147483647 included. It is
+ * clamped rather than honored: a flush occupies the single flusher thread, so a retry
+ * count that large would stall every other stream's delivery behind it. The cap is
+ * well above the documented 1 to 8 and the 9 the probe saw honored. A value that is
+ * not a number falls back to the 3 AWS itself defaults to.
+ *
+ * Read as a BigInteger so the clamp holds at any magnitude: a value too wide even for
+ * a long is still a number, and treating it as unparseable would hand a larger
+ * NumberOfRetries fewer attempts than a smaller one.
+ */
+ private static int numberOfRetries(Processor processor) {
+ String configured = parameterValue(processor, "NumberOfRetries");
+ if (configured == null) {
+ return DEFAULT_NUMBER_OF_RETRIES;
+ }
+ try {
+ return new BigInteger(configured.trim())
+ .max(BigInteger.ZERO)
+ .min(BigInteger.valueOf(MAX_NUMBER_OF_RETRIES))
+ .intValue();
+ } catch (NumberFormatException e) {
+ LOG.warnv("Firehose Lambda processor has a non-numeric NumberOfRetries {0}, using {1}",
+ configured, DEFAULT_NUMBER_OF_RETRIES);
+ return DEFAULT_NUMBER_OF_RETRIES;
+ }
+ }
+
+ private static String parameterValue(Processor processor, String name) {
+ if (processor.getParameters() == null) {
+ return null;
+ }
+ for (ProcessorParameter parameter : processor.getParameters()) {
+ if (parameter != null && name.equals(parameter.getParameterName())) {
+ return parameter.getParameterValue();
+ }
+ }
+ return null;
+ }
+
+ // ── invocation ───────────────────────────────────────────────────────────
+
+ /**
+ * What one batch's invocation settled on: either a function-level failure every
+ * record inherits, or the per-record results the function returned.
+ */
+ private record Invocation(int attemptsMade, boolean functionError,
+ Map<String, LambdaResult> results, Set<String> duplicatedIds) {
+
+ Verdict verdictFor(String recordId) {
+ if (functionError) {
+ return Verdict.failed("Lambda.FunctionError", FUNCTION_ERROR_MESSAGE);
+ }
+ if (duplicatedIds.contains(recordId)) {
+ return Verdict.failed("Lambda.DuplicatedRecordId", DUPLICATED_RECORD_ID_MESSAGE);
+ }
+ LambdaResult result = results.get(recordId);
+ if (result == null) {
+ return Verdict.failed("Lambda.MissingRecordId", MISSING_RECORD_ID_MESSAGE);
+ }
+ if (RESULT_DROPPED.equals(result.result())) {
+ return Verdict.dropped();
+ }
+ if (RESULT_PROCESSING_FAILED.equals(result.result())) {
+ return Verdict.failed("Lambda.ProcessingFailedStatus", PROCESSING_FAILED_MESSAGE);
+ }
+ if (!RESULT_OK.equals(result.result())) {
+ return Verdict.failed("Lambda.InvalidReturnFormat",
+ "The result field must be one of Ok, Dropped or ProcessingFailed.");
+ }
+ if (result.data() == null) {
+ return Verdict.failed("Lambda.InvalidReturnFormat", NULL_DATA_MESSAGE);
+ }
+ try {
+ return Verdict.transformed(Base64.getDecoder().decode(result.data()));
+ } catch (IllegalArgumentException e) {
+ return Verdict.failed("Lambda.InvalidReturnFormat",
+ "The data field must be base64 encoded: " + e.getMessage());
+ }
+ }
+ }
+
+ private record LambdaResult(String result, String data) {}
+
+ /** A transformed payload, a failure, or, with every member null, a drop. */
+ private record Verdict(byte[] data, String errorCode, String errorMessage) {
+
+ static Verdict transformed(byte[] data) {
+ return new Verdict(data, null, null);
+ }
+
+ static Verdict dropped() {
+ return new Verdict(null, null, null);
+ }
+
+ static Verdict failed(String errorCode, String errorMessage) {
+ return new Verdict(null, errorCode, errorMessage);
+ }
+ }
+
+ /**
+ * Invokes the function, retrying only a function-level error: a response the
+ * function returned successfully is taken as final however malformed it is (probed,
+ * a response with no records at all reports attemptsMade 1 while a throwing
+ * function reports NumberOfRetries + 1).
+ */
+ private Invocation invoke(DeliveryStreamDescription stream, String functionArn, List<String> recordIds,
+ List<byte[]> records, Instant deliveryTime, int maxAttempts) {
+ byte[] payload = buildPayload(stream, recordIds, records, deliveryTime);
+ for (int attempt = 1; attempt <= maxAttempts; attempt++) {
+ try {
+ InvokeResult result = lambdaService.invokeArn(functionArn, payload, InvocationType.RequestResponse);
+ if (result.getFunctionError() == null) {
+ return parseResponse(attempt, result.getPayload());
+ }
+ LOG.warnv("Firehose transform {0} returned {1} on attempt {2} of {3}",
+ functionArn, result.getFunctionError(), attempt, maxAttempts);
+ } catch (Exception e) {
+ LOG.warnv("Firehose transform {0} could not be invoked on attempt {1} of {2}: {3}",
+ functionArn, attempt, maxAttempts, e.getMessage());
+ }
+ }
+ return new Invocation(maxAttempts, true, Map.of(), Set.of());
+ }
+
+ private byte[] buildPayload(DeliveryStreamDescription stream, List<String> recordIds,
+ List<byte[]> records, Instant deliveryTime) {
+ Map<String, Object> event = new LinkedHashMap<>();
+ event.put("invocationId", UUID.randomUUID().toString());
+ event.put("deliveryStreamArn", stream.getDeliveryStreamARN());
+ event.put("region", AwsArnUtils.regionOrDefault(stream.getDeliveryStreamARN(),
+ regionResolver.getDefaultRegion()));
+ List<Map<String, Object>> eventRecords = new ArrayList<>(records.size());
+ for (int i = 0; i < records.size(); i++) {
+ Map<String, Object> eventRecord = new LinkedHashMap<>();
+ eventRecord.put("recordId", recordIds.get(i));
+ // Floci does not track per-record arrival, so the batch's delivery time stands
+ // in for all of them, as it does in the error output.
+ eventRecord.put("approximateArrivalTimestamp", deliveryTime.toEpochMilli());
+ eventRecord.put("data", Base64.getEncoder().encodeToString(records.get(i)));
+ eventRecords.add(eventRecord);
+ }
+ event.put("records", eventRecords);
+ try {
+ return mapper.writeValueAsBytes(event);
+ } catch (Exception e) {
+ throw new AwsException("InternalServerException",
+ "Failed to serialize a Firehose transformation payload: " + e.getMessage(), 500);
+ }
+ }
+
+ /**
+ * A response that carries no usable records array leaves every record unaccounted
+ * for, which is what AWS reports for it: Lambda.MissingRecordId per record rather
+ * than a batch-level failure (probed). An entry naming a record the batch never
+ * contained is ignored, also as AWS does.
+ */
+ private Invocation parseResponse(int attempt, byte[] responsePayload) {
+ Map<String, LambdaResult> results = new LinkedHashMap<>();
+ Set<String> duplicatedIds = new HashSet<>();
+ JsonNode records = readRecords(responsePayload);
+ if (records != null) {
+ for (JsonNode record : records) {
+ String recordId = text(record, "recordId");
+ if (recordId == null) {
+ continue;
+ }
+ LambdaResult result = new LambdaResult(text(record, "result"), text(record, "data"));
+ if (results.putIfAbsent(recordId, result) != null) {
+ duplicatedIds.add(recordId);
+ }
+ }
+ }
+ return new Invocation(attempt, false, results, duplicatedIds);
+ }
+
+ private JsonNode readRecords(byte[] responsePayload) {
+ if (responsePayload == null || responsePayload.length == 0) {
+ return null;
+ }
+ try {
+ JsonNode response = mapper.readTree(responsePayload);
+ JsonNode records = response == null ? null : response.get("records");
+ return records != null && records.isArray() ? records : null;
+ } catch (Exception e) {
+ LOG.warnv("Firehose transform returned a response that is not JSON: {0}", e.getMessage());
+ return null;
+ }
+ }
+
+ private static String text(JsonNode node, String field) {
+ JsonNode value = node.get(field);
+ return value == null || !value.isTextual() ? null : value.asText();
+ }
+
+ // ── error output ─────────────────────────────────────────────────────────
+
+ private record FailedRecord(byte[] data, String errorCode, String errorMessage) {}
+
+ /**
+ * One NDJSON line per failed record, in the shape and member order real AWS writes
+ * (probed). It is not the format-conversion error object: that one leads with
+ * attemptsMade and carries lastErrorCode and a dataCatalogTable block, where this
+ * one leads with rawData and ends with lambdaARN, in that casing.
+ *
+ * Both timestamps are the delivery time, a deviation: Floci tracks no per-record
+ * arrival. attemptsMade counts the batch's attempts, a record never having retries
+ * of its own.
+ */
+ private String writeErrorOutput(DeliveryStreamDescription stream, S3Destination s3, String bucket,
+ List<FailedRecord> failures, Instant deliveryTime,
+ String functionArn, int attemptsMade) {
+ String errorKey = S3ObjectKeyResolver.resolveErrorKey(s3, stream.getDeliveryStreamName(),
+ stream.getVersionId(), deliveryTime, ERROR_OUTPUT_TYPE);
+ ByteArrayOutputStream body = new ByteArrayOutputStream();
+ for (FailedRecord failure : failures) {
+ Map<String, Object> line = new LinkedHashMap<>();
+ line.put("rawData", Base64.getEncoder().encodeToString(failure.data()));
+ line.put("errorCode", failure.errorCode());
+ line.put("errorMessage", failure.errorMessage());
+ line.put("attemptsMade", attemptsMade);
+ line.put("arrivalTimestamp", deliveryTime.toEpochMilli());
+ line.put("attemptEndingTimestamp", deliveryTime.toEpochMilli());
+ line.put("lambdaARN", functionArn);
+ try {
+ body.write(mapper.writeValueAsBytes(line));
+ body.write('\n');
+ } catch (Exception e) {
+ throw new AwsException("InternalServerException",
+ "Failed to serialize a Firehose error-output record: " + e.getMessage(), 500);
+ }
+ }
+ s3Service.putObject(bucket, errorKey, body.toByteArray(), CONTENT_TYPE, Map.of());
+ return errorKey;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
index 0587e25e24..86990d586c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
@@ -102,6 +102,7 @@ public class FirehoseService implements ResourceProvider {
// commits exactly the position that snapshot covered.
private final Map<String, Map<String, String>> pendingSourceIterators = new ConcurrentHashMap<>();
private final FirehoseParquetConverter parquetConverter;
+ private final FirehoseLambdaTransformer lambdaTransformer;
private final Clock clock;
private final long tickIntervalSeconds;
private final int flushRecordCount;
@@ -187,7 +188,8 @@ public class FirehoseService implements ResourceProvider {
@Inject
public FirehoseService(StorageFactory storageFactory, S3Service s3Service, KinesisService kinesisService,
RegionResolver regionResolver, Clock clock, EmulatorConfig config,
- FirehoseParquetConverter parquetConverter) {
+ FirehoseParquetConverter parquetConverter,
+ FirehoseLambdaTransformer lambdaTransformer) {
this.streamStore = storageFactory.create("firehose", "streams.json",
new TypeReference<Map<String, DeliveryStreamDescription>>() {});
this.sourceIteratorStore = storageFactory.create("firehose", "source-iterators.json",
@@ -196,6 +198,7 @@ public class FirehoseService implements ResourceProvider {
this.kinesisService = kinesisService;
this.regionResolver = regionResolver;
this.parquetConverter = parquetConverter;
+ this.lambdaTransformer = lambdaTransformer;
this.clock = clock;
this.tickIntervalSeconds = Math.max(1, config.services().firehose().tickIntervalSeconds());
this.flushRecordCount = Math.max(0, config.services().firehose().flushRecordCount());
@@ -313,7 +316,6 @@ public class FirehoseService implements ResourceProvider {
if (s3Config != null) {
s3Config.canonicalizeProcessors();
}
- warnIfProcessingEnabled(name, s3Config);
String arn = AwsArnUtils.Arn.of("firehose", region, accountId,
"deliverystream/" + name).toString();
// CreateDeliveryStream's KinesisStreamSourceConfiguration carries only the ARN and
@@ -392,7 +394,6 @@ public class FirehoseService implements ResourceProvider {
stream.setLastUpdateTimestamp(java.time.Instant.now());
streamPut(streamKey, stream);
LOG.infov("Updated destination {0} of Firehose delivery stream {1}", destinationId, name);
- warnIfProcessingEnabled(name, stream.s3Destination());
}
public void startDeliveryStreamEncryption(String name, String keyType, String keyArn) {
@@ -464,20 +465,6 @@ public class FirehoseService implements ResourceProvider {
}
}
- /**
- * Says, where the configuration is set, that the transformation will not be applied.
- * Fires on create and on every update that leaves it enabled, so a caller who keeps
- * changing the destination keeps being told. Deliberately not in the flush path,
- * which runs on every buffered delivery: create and update are caller-driven and are
- * the moments a caller can act on the warning.
- */
- private static void warnIfProcessingEnabled(String name, S3Destination s3) {
- if (s3 != null && s3.isProcessingEnabled()) {
- LOG.warnv("Delivery stream {0} enables a record transformation, which Floci does not"
- + " apply yet; its records will be delivered untransformed", name);
- }
- }
-
private static void mergeDestination(S3Destination current, S3Destination update) {
if (update.getRoleArn() != null) current.setRoleArn(update.getRoleArn());
if (update.getBucketArn() != null) current.setBucketArn(update.getBucketArn());
@@ -893,10 +880,26 @@ public class FirehoseService implements ResourceProvider {
try {
String bucket = resolveBucket(stream);
S3Destination s3 = stream.s3Destination();
+ List<byte[]> records = toFlush;
+ if (s3 != null && s3.isProcessingEnabled()) {
+ ensureBucket(bucket);
+ FirehoseLambdaTransformer.Outcome transformed =
+ lambdaTransformer.transform(stream, bucket, records, clock.instant());
+ LOG.infov("Transformed {0} records from stream {1} ({2} dropped, {3} failed)",
+ records.size(), streamName, transformed.droppedRecords(), transformed.failedRecords());
+ records = transformed.records();
+ if (records.isEmpty()) {
+ // Nothing survived the transform. The batch is accounted for, dropped
+ // records deliberately leaving no trace and failed ones already in the
+ // error output, so the source may advance past it.
+ commitSourceIterators(streamName, checkpoint);
+ return;
+ }
+ }
if (s3 != null && s3.isDataFormatConversionEnabled()) {
ensureBucket(bucket);
FirehoseParquetConverter.Outcome outcome =
- parquetConverter.deliver(stream, bucket, toFlush, clock.instant());
+ parquetConverter.deliver(stream, bucket, records, clock.instant());
LOG.infov("Converted {0} records ({1} failed) from stream {2} to s3://{3}/{4}",
outcome.convertedRecords(), outcome.failedRecords(), streamName, bucket,
outcome.dataKey() != null ? outcome.dataKey() : outcome.errorKey());
@@ -917,7 +920,7 @@ public class FirehoseService implements ResourceProvider {
// (verified: three "abc" records arrive as the 9 bytes "abcabcabc").
// See the deviation noted in docs/services/firehose.md.
ByteArrayOutputStream payload = new ByteArrayOutputStream();
- for (byte[] data : toFlush) {
+ for (byte[] data : records) {
payload.writeBytes(data);
if (data.length > 0 && data[data.length - 1] != '\n') {
payload.write('\n');
@@ -928,7 +931,7 @@ public class FirehoseService implements ResourceProvider {
s3Service.putObject(bucket, key, body, "application/octet-stream", Map.of(),
new PutObjectOptions().withContentEncoding(compression.contentEncoding()));
LOG.infov("Flushed {0} records from stream {1} to s3://{2}/{3} ({4})",
- toFlush.size(), streamName, bucket, key, compression.wireValue());
+ records.size(), streamName, bucket, key, compression.wireValue());
// Only now: the records these iterators were read past are durable.
commitSourceIterators(streamName, checkpoint);
} catch (Exception e) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
index 163982a4e0..5d8866ca76 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java
@@ -263,17 +263,18 @@ public class DeliveryStreamDescription {
}
/**
- * True when a transformation applies to deliveries. An omitted Enabled is taken
- * as enabled, following the conversion block, where that was probed; the
- * processing block's own behavior for an omitted Enabled was not, and nothing
- * but the "not applied yet" warning reads this today.
+ * True when a transformation applies to deliveries: the configuration is present
+ * and Enabled is explicitly true. An omitted Enabled leaves it disabled, the
+ * opposite of the conversion block below. Probed rather than carried over from
+ * there (2026-09-11): DescribeDeliveryStream echoes an omitted Enabled as false
+ * here and as true on the conversion block.
*
* Ignored for serialization for the same reason as the accessor below.
*/
@JsonIgnore
public boolean isProcessingEnabled() {
return processingConfiguration != null
- && !Boolean.FALSE.equals(processingConfiguration.getEnabled());
+ && Boolean.TRUE.equals(processingConfiguration.getEnabled());
}
/**
@@ -326,13 +327,25 @@ public class DeliveryStreamDescription {
bufferingHints.setIntervalInSeconds(BufferingHints.DEFAULT_INTERVAL_SECONDS);
}
}
+ // Legacy persisted state again: a processing block stored before Enabled was
+ // defaulted never passes canonicalizeProcessors a second time, and would be
+ // described without the member AWS always returns. Only Enabled is healed
+ // here, since the parameters AWS defaults include RoleArn, which a read must
+ // not inject from whatever role is current by then.
+ if (processingConfiguration != null && processingConfiguration.getEnabled() == null) {
+ processingConfiguration.setEnabled(false);
+ }
}
/**
* Real AWS answers with more than the caller sent: a Lambda processor comes back
- * with NumberOfRetries defaulted to 3 and RoleArn filled from the delivery
- * stream's role, and the parameters are echoed in a fixed order whatever order
- * they arrived in (probed 2026-09-10).
+ * with NumberOfRetries defaulted to 3, RoleArn filled from the delivery stream's
+ * role, and BufferSizeInMBs and BufferIntervalInSeconds defaulted to 1 and 60,
+ * and the parameters are echoed in a fixed order whatever order they arrived in
+ * (probed 2026-09-10 and 2026-09-11). The two buffer defaults are the Lambda
+ * processor's own, not the destination's BufferingHints: a destination buffering
+ * 5 MiB over 300s still echoes 1 and 60 here. An omitted Enabled is stored as
+ * false, which is what leaves the transformation disabled.
*
* Called as a destination is written, not as it is read. Doing it on the way out
* would mutate what storage handed back, so whether a Describe had happened would
@@ -340,7 +353,13 @@ public class DeliveryStreamDescription {
* role was current at the time of the read.
*/
public void canonicalizeProcessors() {
- if (processingConfiguration == null || processingConfiguration.getProcessors() == null) {
+ if (processingConfiguration == null) {
+ return;
+ }
+ if (processingConfiguration.getEnabled() == null) {
+ processingConfiguration.setEnabled(false);
+ }
+ if (processingConfiguration.getProcessors() == null) {
return;
}
for (Processor processor : processingConfiguration.getProcessors()) {
@@ -360,6 +379,8 @@ public class DeliveryStreamDescription {
if (roleArn != null) {
byName.putIfAbsent("RoleArn", roleArn);
}
+ byName.putIfAbsent("BufferSizeInMBs", "1");
+ byName.putIfAbsent("BufferIntervalInSeconds", "60");
List<ProcessorParameter> ordered = new ArrayList<>(byName.size());
for (String name : LAMBDA_PARAMETER_ORDER) {
if (byName.containsKey(name)) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionTest.java
index 9f9d733273..a6cf8fd15f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionTest.java
@@ -15,6 +15,9 @@ import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescript
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OrcSerDe;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OutputFormatConfiguration;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ParquetSerDe;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.SchemaConfiguration;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Serializer;
@@ -23,6 +26,7 @@ import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.testing.MutableClock;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import java.util.HashMap;
@@ -59,6 +63,7 @@ class FirehoseDataFormatConversionTest {
private FirehoseService service;
private FirehoseParquetConverter parquetConverter;
+ private FirehoseLambdaTransformer lambdaTransformer;
private S3Service s3Service;
@BeforeEach
@@ -81,10 +86,11 @@ class FirehoseDataFormatConversionTest {
when(regionResolver.getDefaultRegion()).thenReturn("us-east-1");
when(regionResolver.getAccountId()).thenReturn("000000000000");
parquetConverter = Mockito.mock(FirehoseParquetConverter.class);
+ lambdaTransformer = Mockito.mock(FirehoseLambdaTransformer.class);
s3Service = Mockito.mock(S3Service.class);
service = new FirehoseService(storageFactory, s3Service,
Mockito.mock(KinesisService.class), regionResolver, new MutableClock(), config,
- parquetConverter);
+ parquetConverter, lambdaTransformer);
}
private static SchemaConfiguration schema() {
@@ -544,6 +550,57 @@ class FirehoseDataFormatConversionTest {
any(byte[].class), anyString(), Mockito.anyMap(), any());
}
+ /**
+ * A stream configured for both has to convert what the transform returned, not what
+ * was put: the two were wired in separately, so nothing else would catch a delivery
+ * that passed the original bytes to the converter.
+ */
+ @Test
+ void conversionOfATransformingStreamConvertsWhatTheTransformReturned() {
+ Mockito.when(lambdaTransformer.transform(any(), anyString(), any(), any()))
+ .thenReturn(new FirehoseLambdaTransformer.Outcome(
+ List.of("{\"ticker\": \"BBB\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8)),
+ 0, 0, null));
+ Mockito.when(parquetConverter.deliver(any(), anyString(), any(), any()))
+ .thenReturn(new FirehoseParquetConverter.Outcome(1, 0, "data/key.parquet", null));
+ service.createDeliveryStream("stream", destination(s3 -> s3.setProcessingConfiguration(lambdaProcessing())));
+ service.putRecord("stream", record("{\"ticker\": \"AAA\"}"));
+
+ service.flush("stream");
+
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor<List<byte[]>> converted = ArgumentCaptor.forClass(List.class);
+ Mockito.verify(parquetConverter).deliver(any(), Mockito.eq("results"), converted.capture(), any());
+ assertEquals(1, converted.getValue().size());
+ assertEquals("{\"ticker\": \"BBB\"}",
+ new String(converted.getValue().get(0), java.nio.charset.StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void aTransformThatKeptNothingNeverReachesTheConverter() {
+ Mockito.when(lambdaTransformer.transform(any(), anyString(), any(), any()))
+ .thenReturn(new FirehoseLambdaTransformer.Outcome(List.of(), 1, 0, null));
+ service.createDeliveryStream("stream", destination(s3 -> s3.setProcessingConfiguration(lambdaProcessing())));
+ service.putRecord("stream", record("{\"ticker\": \"AAA\"}"));
+
+ service.flush("stream");
+
+ Mockito.verify(parquetConverter, Mockito.never()).deliver(any(), anyString(), any(), any());
+ }
+
+ private static ProcessingConfiguration lambdaProcessing() {
+ ProcessorParameter lambdaArn = new ProcessorParameter();
+ lambdaArn.setParameterName("LambdaArn");
+ lambdaArn.setParameterValue("arn:aws:lambda:us-east-1:000000000000:function:transform");
+ Processor processor = new Processor();
+ processor.setType("Lambda");
+ processor.setParameters(List.of(lambdaArn));
+ ProcessingConfiguration processing = new ProcessingConfiguration();
+ processing.setEnabled(true);
+ processing.setProcessors(List.of(processor));
+ return processing;
+ }
+
private static io.github.hectorvent.floci.services.firehose.model.Record record(String json) {
io.github.hectorvent.floci.services.firehose.model.Record record =
new io.github.hectorvent.floci.services.firehose.model.Record();
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformIntegrationTest.java
new file mode 100644
index 0000000000..28ad6b293d
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformIntegrationTest.java
@@ -0,0 +1,246 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.InjectMock;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+
+/**
+ * End-to-end coverage of the Lambda transform on the real delivery path: what
+ * the function returns is what reaches S3, and what it fails reaches the error
+ * output instead. {@link FirehoseLambdaTransformerTest} covers the per-record
+ * result semantics against the transformer directly; this class is what proves
+ * the flush path runs it at all.
+ *
+ * The function itself is mocked. Running a real one needs Docker, which a
+ * delivery-path test should not depend on, and the invocation payload is
+ * asserted here from what the mock received.
+ */
+@QuarkusTest
+@TestProfile(FirehoseLambdaTransformIntegrationTest.FlushEveryPairProfile.class)
+class FirehoseLambdaTransformIntegrationTest {
+
+ public static class FlushEveryPairProfile implements QuarkusTestProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ // Each test puts a batch of two, which then delivers inline.
+ return Map.of("floci.services.firehose.flush-record-count", "2");
+ }
+ }
+
+ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static final String TARGET_PREFIX = "Firehose_20150804.";
+ private static final String FUNCTION_ARN = "arn:aws:lambda:us-east-1:000000000000:function:transform";
+ private static final Pattern KEY_PATTERN = Pattern.compile("<Key>([^<]+)</Key>");
+
+ @InjectMock
+ LambdaService lambdaService;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void theFunctionsOutputIsWhatReachesTheDestination() {
+ String stream = "transform-ok-stream";
+ String bucket = "transform-ok-archive";
+ createDeliveryStream(stream, bucket);
+ respondWith((recordId, index) -> "{\"recordId\":\"" + recordId + "\",\"result\":\"Ok\",\"data\":\""
+ + encode("TRANSFORMED-" + index) + "\"}");
+
+ putRecordBatch(stream, "one", "two");
+
+ given()
+ .when()
+ .get("/" + bucket + "/" + firstKey(bucket, "data/"))
+ .then()
+ .statusCode(200)
+ .body(equalTo("TRANSFORMED-0\nTRANSFORMED-1\n"));
+ }
+
+ @Test
+ void aFailedRecordGoesToTheErrorOutputWhileTheRestDelivers() {
+ String stream = "transform-failed-stream";
+ String bucket = "transform-failed-archive";
+ createDeliveryStream(stream, bucket);
+ respondWith((recordId, index) -> index == 0
+ ? "{\"recordId\":\"" + recordId + "\",\"result\":\"ProcessingFailed\"}"
+ : "{\"recordId\":\"" + recordId + "\",\"result\":\"Ok\",\"data\":\"" + encode("kept") + "\"}");
+
+ putRecordBatch(stream, "one", "two");
+
+ given()
+ .when()
+ .get("/" + bucket + "/" + firstKey(bucket, "data/"))
+ .then()
+ .statusCode(200)
+ .body(equalTo("kept\n"));
+
+ String errorLine = given()
+ .when()
+ .get("/" + bucket + "/" + firstKey(bucket, "errors/processing-failed/"))
+ .then()
+ .statusCode(200)
+ .extract().asString();
+ assertTrue(errorLine.contains("\"errorCode\":\"Lambda.ProcessingFailedStatus\""), errorLine);
+ assertTrue(errorLine.contains("\"rawData\":\"" + encode("one") + "\""), errorLine);
+ assertTrue(errorLine.contains("\"lambdaARN\":\"" + FUNCTION_ARN + "\""), errorLine);
+ }
+
+ @Test
+ void aBatchTheFunctionDroppedEntirelyDeliversNothing() {
+ String stream = "transform-dropped-stream";
+ String bucket = "transform-dropped-archive";
+ createDeliveryStream(stream, bucket);
+ respondWith((recordId, index) -> "{\"recordId\":\"" + recordId + "\",\"result\":\"Dropped\"}");
+
+ putRecordBatch(stream, "one", "two");
+
+ String listing = given()
+ .when()
+ .get("/" + bucket)
+ .then()
+ .statusCode(200)
+ .extract().asString();
+ assertFalse(KEY_PATTERN.matcher(listing).find(), "expected no delivered object, got: " + listing);
+ }
+
+ @Test
+ void theInvocationCarriesEveryBufferedRecord() {
+ String stream = "transform-payload-stream";
+ String bucket = "transform-payload-archive";
+ createDeliveryStream(stream, bucket);
+ List<byte[]> payloads = respondWith((recordId, index) ->
+ "{\"recordId\":\"" + recordId + "\",\"result\":\"Dropped\"}");
+
+ putRecordBatch(stream, "one", "two");
+
+ assertEquals(1, payloads.size(), "the whole buffer should be one invocation");
+ String payload = new String(payloads.get(0), StandardCharsets.UTF_8);
+ assertTrue(payload.contains("\"deliveryStreamArn\":\"arn:aws:firehose:"), payload);
+ assertTrue(payload.contains("\"data\":\"" + encode("one") + "\""), payload);
+ assertTrue(payload.contains("\"data\":\"" + encode("two") + "\""), payload);
+ }
+
+ /**
+ * Answers each invocation from the record ids it actually carries, which the caller
+ * cannot name: they are minted per batch. Returns the payloads the mock received.
+ */
+ private List<byte[]> respondWith(RecordResponder responder) {
+ List<byte[]> payloads = new ArrayList<>();
+ Mockito.when(lambdaService.invokeArn(anyString(), any(), any())).thenAnswer(invocation -> {
+ byte[] payload = invocation.getArgument(1);
+ payloads.add(payload);
+ List<String> entries = new ArrayList<>();
+ Matcher recordIds = Pattern.compile("\"recordId\":\"([^\"]+)\"")
+ .matcher(new String(payload, StandardCharsets.UTF_8));
+ for (int index = 0; recordIds.find(); index++) {
+ entries.add(responder.respond(recordIds.group(1), index));
+ }
+ InvokeResult result = new InvokeResult();
+ result.setStatusCode(200);
+ result.setPayload(("{\"records\":[" + String.join(",", entries) + "]}")
+ .getBytes(StandardCharsets.UTF_8));
+ return result;
+ });
+ return payloads;
+ }
+
+ @FunctionalInterface
+ private interface RecordResponder {
+ String respond(String recordId, int index);
+ }
+
+ private static String encode(String body) {
+ return Base64.getEncoder().encodeToString(body.getBytes(StandardCharsets.UTF_8));
+ }
+
+ private void createDeliveryStream(String streamName, String bucket) {
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")
+ .body("""
+ {
+ "DeliveryStreamName": "%s",
+ "DeliveryStreamType": "DirectPut",
+ "ExtendedS3DestinationConfiguration": {
+ "RoleARN": "arn:aws:iam::000000000000:role/firehose-delivery-role",
+ "BucketARN": "arn:aws:s3:::%s",
+ "Prefix": "data/!{timestamp:yyyy}/",
+ "ErrorOutputPrefix": "errors/!{firehose:error-output-type}/",
+ "ProcessingConfiguration": {
+ "Enabled": true,
+ "Processors": [
+ {
+ "Type": "Lambda",
+ "Parameters": [{"ParameterName": "LambdaArn", "ParameterValue": "%s"}]
+ }
+ ]
+ }
+ }
+ }
+ """.formatted(streamName, bucket, FUNCTION_ARN))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DeliveryStreamARN", notNullValue());
+ }
+
+ private void putRecordBatch(String streamName, String... bodies) {
+ List<String> records = new ArrayList<>();
+ for (String body : bodies) {
+ records.add("{\"Data\": \"" + encode(body) + "\"}");
+ }
+ given()
+ .contentType(CONTENT_TYPE)
+ .header("X-Amz-Target", TARGET_PREFIX + "PutRecordBatch")
+ .body("{ \"DeliveryStreamName\": \"" + streamName + "\", \"Records\": ["
+ + String.join(",", records) + "] }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("FailedPutCount", equalTo(0));
+ }
+
+ private String firstKey(String bucket, String prefix) {
+ String listing = given()
+ .when()
+ .get("/" + bucket)
+ .then()
+ .statusCode(200)
+ .extract().asString();
+ Matcher key = KEY_PATTERN.matcher(listing);
+ while (key.find()) {
+ if (key.group(1).startsWith(prefix)) {
+ return key.group(1);
+ }
+ }
+ throw new AssertionError("expected an object under " + prefix + " in " + bucket + ", got: " + listing);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformerTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformerTest.java
new file mode 100644
index 0000000000..ebda90bbee
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformerTest.java
@@ -0,0 +1,513 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
+import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.lambda.model.InvocationType;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyMap;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+/**
+ * The result semantics, error codes and error-object shape asserted here were
+ * probed against real AWS (us-west-2, 2026-09-10), including which failures are
+ * retried: a function that errors is, a response the function shaped wrongly is
+ * not.
+ */
+class FirehoseLambdaTransformerTest {
+
+ private static final String BUCKET = "results";
+ private static final String FUNCTION_ARN = "arn:aws:lambda:us-east-1:000000000000:function:transform";
+ private static final Instant DELIVERY_TIME = Instant.parse("2026-01-01T00:00:00Z");
+
+ private LambdaService lambdaService;
+ private S3Service s3Service;
+ private FirehoseLambdaTransformer transformer;
+ private final ObjectMapper mapper = new ObjectMapper();
+ private JsonNode invocationEvent;
+
+ @BeforeEach
+ void setUp() {
+ lambdaService = mock(LambdaService.class);
+ s3Service = mock(S3Service.class);
+ transformer = new FirehoseLambdaTransformer(lambdaService, s3Service, mapper,
+ new RegionResolver("us-east-1", "000000000000"));
+ }
+
+ // ── fixtures ─────────────────────────────────────────────────────────────
+
+ private static DeliveryStreamDescription stream(Processor... processors) {
+ ProcessingConfiguration processing = new ProcessingConfiguration();
+ processing.setEnabled(true);
+ processing.setProcessors(List.of(processors));
+ S3Destination s3 = new S3Destination();
+ s3.setBucketArn("arn:aws:s3:::" + BUCKET);
+ s3.setErrorOutputPrefix("errors/!{firehose:error-output-type}/");
+ s3.setProcessingConfiguration(processing);
+ return new DeliveryStreamDescription("stream", "arn:aws:firehose:us-west-2:000000000000:deliverystream/stream", s3);
+ }
+
+ private static Processor lambdaProcessor(String... parameters) {
+ List<ProcessorParameter> list = new ArrayList<>();
+ list.add(parameter("LambdaArn", FUNCTION_ARN));
+ for (int i = 0; i < parameters.length; i += 2) {
+ list.add(parameter(parameters[i], parameters[i + 1]));
+ }
+ Processor processor = new Processor();
+ processor.setType("Lambda");
+ processor.setParameters(list);
+ return processor;
+ }
+
+ private static ProcessorParameter parameter(String name, String value) {
+ ProcessorParameter parameter = new ProcessorParameter();
+ parameter.setParameterName(name);
+ parameter.setParameterValue(value);
+ return parameter;
+ }
+
+ private static byte[] record(String body) {
+ return body.getBytes(StandardCharsets.UTF_8);
+ }
+
+ private static String encode(String body) {
+ return Base64.getEncoder().encodeToString(record(body));
+ }
+
+ /**
+ * Answers the invocation from the record ids the transformer actually generated,
+ * which is the only way a test can name them: they are minted per batch.
+ */
+ private void replyWith(Function<List<String>, String> responder) {
+ when(lambdaService.invokeArn(anyString(), any(), any())).thenAnswer(invocation -> {
+ invocationEvent = mapper.readTree((byte[]) invocation.getArgument(1));
+ List<String> recordIds = new ArrayList<>();
+ invocationEvent.get("records").forEach(node -> recordIds.add(node.get("recordId").asText()));
+ InvokeResult result = new InvokeResult();
+ result.setStatusCode(200);
+ result.setPayload(responder.apply(recordIds).getBytes(StandardCharsets.UTF_8));
+ return result;
+ });
+ }
+
+ private static String records(String... entries) {
+ return "{\"records\":[" + String.join(",", entries) + "]}";
+ }
+
+ private static String ok(String recordId, String data) {
+ return "{\"recordId\":\"" + recordId + "\",\"result\":\"Ok\",\"data\":\"" + encode(data) + "\"}";
+ }
+
+ private static String outcome(String recordId, String result) {
+ return "{\"recordId\":\"" + recordId + "\",\"result\":\"" + result + "\"}";
+ }
+
+ private List<String> delivered(FirehoseLambdaTransformer.Outcome outcome) {
+ return outcome.records().stream()
+ .map(data -> new String(data, StandardCharsets.UTF_8))
+ .collect(Collectors.toList());
+ }
+
+ private List<JsonNode> errorLines() throws Exception {
+ ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service).putObject(eq(BUCKET), key.capture(), body.capture(),
+ eq("application/octet-stream"), anyMap());
+ assertTrue(key.getValue().startsWith("errors/processing-failed/"),
+ "error output key was " + key.getValue());
+ List<JsonNode> lines = new ArrayList<>();
+ for (String line : new String(body.getValue(), StandardCharsets.UTF_8).split("\n")) {
+ lines.add(mapper.readTree(line));
+ }
+ return lines;
+ }
+
+ private FirehoseLambdaTransformer.Outcome transform(DeliveryStreamDescription stream, byte[]... records) {
+ return transformer.transform(stream, BUCKET, List.of(records), DELIVERY_TIME);
+ }
+
+ // ── pass-through ─────────────────────────────────────────────────────────
+
+ @Test
+ void aStreamWithoutAProcessingConfigurationIsNotTransformed() {
+ S3Destination s3 = new S3Destination();
+ s3.setBucketArn("arn:aws:s3:::" + BUCKET);
+ DeliveryStreamDescription stream =
+ new DeliveryStreamDescription("stream", "arn:aws:firehose:::stream", s3);
+ List<byte[]> records = List.of(record("a"));
+
+ FirehoseLambdaTransformer.Outcome outcome =
+ transformer.transform(stream, BUCKET, records, DELIVERY_TIME);
+
+ assertSame(records, outcome.records());
+ verifyNoInteractions(lambdaService);
+ }
+
+ /**
+ * Probed 2026-09-11: an omitted Enabled is stored as false, so it leaves the stream
+ * untransformed. The conversion block treats an omitted Enabled the other way, which
+ * is why this is asserted rather than inferred from it.
+ */
+ @Test
+ void aProcessingConfigurationWithoutAnExplicitEnabledIsNotTransformed() {
+ DeliveryStreamDescription stream = stream(lambdaProcessor());
+ stream.s3Destination().getProcessingConfiguration().setEnabled(null);
+
+ FirehoseLambdaTransformer.Outcome outcome = transform(stream, record("a"));
+
+ assertEquals(List.of("a"), delivered(outcome));
+ verifyNoInteractions(lambdaService);
+ }
+
+ @Test
+ void aDisabledProcessingConfigurationIsNotTransformed() {
+ DeliveryStreamDescription stream = stream(lambdaProcessor());
+ stream.s3Destination().getProcessingConfiguration().setEnabled(false);
+
+ transform(stream, record("a"));
+
+ verifyNoInteractions(lambdaService);
+ }
+
+ @Test
+ void aProcessorTypeFlociDoesNotApplyLeavesTheRecordsAlone() {
+ Processor appendDelimiter = new Processor();
+ appendDelimiter.setType("AppendDelimiterToRecord");
+
+ FirehoseLambdaTransformer.Outcome outcome = transform(stream(appendDelimiter), record("a"));
+
+ assertEquals(List.of("a"), delivered(outcome));
+ verifyNoInteractions(lambdaService);
+ }
+
+ // ── invocation payload ───────────────────────────────────────────────────
+
+ @Test
+ void theInvocationCarriesTheProbedEventShape() {
+ replyWith(ids -> records(ok(ids.get(0), "transformed")));
+
+ transform(stream(lambdaProcessor()), record("original"));
+
+ assertEquals(List.of("invocationId", "deliveryStreamArn", "region", "records"),
+ fieldNames(invocationEvent));
+ assertEquals("arn:aws:firehose:us-west-2:000000000000:deliverystream/stream",
+ invocationEvent.get("deliveryStreamArn").asText());
+ assertEquals("us-west-2", invocationEvent.get("region").asText());
+ JsonNode record = invocationEvent.get("records").get(0);
+ assertEquals(List.of("recordId", "approximateArrivalTimestamp", "data"), fieldNames(record));
+ assertEquals(DELIVERY_TIME.toEpochMilli(), record.get("approximateArrivalTimestamp").asLong());
+ assertEquals(encode("original"), record.get("data").asText());
+ verify(lambdaService).invokeArn(eq(FUNCTION_ARN), any(), eq(InvocationType.RequestResponse));
+ }
+
+ private static List<String> fieldNames(JsonNode node) {
+ List<String> names = new ArrayList<>();
+ node.fieldNames().forEachRemaining(names::add);
+ return names;
+ }
+
+ // ── per-record results ───────────────────────────────────────────────────
+
+ @Test
+ void okReplacesTheRecordWithTheReturnedData() {
+ replyWith(ids -> records(ok(ids.get(0), "one"), ok(ids.get(1), "two")));
+
+ FirehoseLambdaTransformer.Outcome outcome =
+ transform(stream(lambdaProcessor()), record("a"), record("b"));
+
+ assertEquals(List.of("one", "two"), delivered(outcome));
+ assertEquals(0, outcome.failedRecords());
+ assertNull(outcome.errorKey());
+ verify(s3Service, never()).putObject(anyString(), anyString(), any(), anyString(), anyMap());
+ }
+
+ @Test
+ void droppedLeavesNoRecordAndNoErrorObject() {
+ replyWith(ids -> records(outcome(ids.get(0), "Dropped"), ok(ids.get(1), "kept")));
+
+ FirehoseLambdaTransformer.Outcome outcome =
+ transform(stream(lambdaProcessor()), record("a"), record("b"));
+
+ assertEquals(List.of("kept"), delivered(outcome));
+ assertEquals(1, outcome.droppedRecords());
+ assertEquals(0, outcome.failedRecords());
+ assertNull(outcome.errorKey());
+ }
+
+ @Test
+ void processingFailedRoutesOnlyThatRecordToTheErrorOutput() throws Exception {
+ replyWith(ids -> records(outcome(ids.get(0), "ProcessingFailed"), ok(ids.get(1), "kept")));
+
+ FirehoseLambdaTransformer.Outcome outcome =
+ transform(stream(lambdaProcessor()), record("a"), record("b"));
+
+ assertEquals(List.of("kept"), delivered(outcome));
+ assertEquals(1, outcome.failedRecords());
+ JsonNode line = errorLines().get(0);
+ assertEquals("Lambda.ProcessingFailedStatus", line.get("errorCode").asText());
+ assertEquals("ProcessingFailed status set for record", line.get("errorMessage").asText());
+ assertEquals(encode("a"), line.get("rawData").asText());
+ }
+
+ @Test
+ void aRecordTheFunctionDidNotReturnFails() throws Exception {
+ replyWith(ids -> records(ok(ids.get(0), "one")));
+
+ transform(stream(lambdaProcessor()), record("a"), record("b"));
+
+ JsonNode line = errorLines().get(0);
+ assertEquals("Lambda.MissingRecordId", line.get("errorCode").asText());
+ assertEquals(encode("b"), line.get("rawData").asText());
+ }
+
+ @Test
+ void aRecordIdReturnedTwiceFails() throws Exception {
+ replyWith(ids -> records(ok(ids.get(0), "one"), ok(ids.get(0), "again"), ok(ids.get(1), "two")));
+
+ FirehoseLambdaTransformer.Outcome outcome =
+ transform(stream(lambdaProcessor()), record("a"), record("b"));
+
+ assertEquals(List.of("two"), delivered(outcome));
+ assertEquals("Lambda.DuplicatedRecordId", errorLines().get(0).get("errorCode").asText());
+ }
+
+ @Test
+ void aRecordIdTheBatchNeverCarriedIsIgnored() {
+ replyWith(ids -> records(ok(ids.get(0), "one"), ok("not-in-this-batch", "stray")));
+
+ FirehoseLambdaTransformer.Outcome outcome = transform(stream(lambdaProcessor()), record("a"));
+
+ assertEquals(List.of("one"), delivered(outcome));
+ assertEquals(0, outcome.failedRecords());
+ }
+
+ @Test
+ void okWithoutDataFails() throws Exception {
+ replyWith(ids -> records(outcome(ids.get(0), "Ok")));
+
+ transform(stream(lambdaProcessor()), record("a"));
+
+ JsonNode line = errorLines().get(0);
+ assertEquals("Lambda.InvalidReturnFormat", line.get("errorCode").asText());
+ assertTrue(line.get("errorMessage").asText().startsWith("The data field cannot be null"));
+ }
+
+ /**
+ * The message here is Floci's own: AWS was not probed for a data field that is not
+ * base64 at all, only for one that is absent.
+ */
+ @Test
+ void dataThatIsNotBase64Fails() throws Exception {
+ replyWith(ids -> records(
+ "{\"recordId\":\"" + ids.get(0) + "\",\"result\":\"Ok\",\"data\":\"not base64 %%%\"}",
+ ok(ids.get(1), "kept")));
+
+ FirehoseLambdaTransformer.Outcome outcome =
+ transform(stream(lambdaProcessor()), record("a"), record("b"));
+
+ assertEquals(List.of("kept"), delivered(outcome));
+ assertEquals(1, outcome.failedRecords());
+ JsonNode line = errorLines().get(0);
+ assertEquals("Lambda.InvalidReturnFormat", line.get("errorCode").asText());
+ assertTrue(line.get("errorMessage").asText().startsWith("The data field must be base64 encoded"));
+ assertEquals(encode("a"), line.get("rawData").asText());
+ }
+
+ @Test
+ void anUnknownResultValueFails() throws Exception {
+ replyWith(ids -> records(outcome(ids.get(0), "Maybe")));
+
+ transform(stream(lambdaProcessor()), record("a"));
+
+ assertEquals("Lambda.InvalidReturnFormat", errorLines().get(0).get("errorCode").asText());
+ }
+
+ // ── failures of the invocation itself ────────────────────────────────────
+
+ @Test
+ void aFunctionErrorIsRetriedAndThenFailsTheWholeBatch() throws Exception {
+ InvokeResult errored = new InvokeResult();
+ errored.setStatusCode(200);
+ errored.setFunctionError("Unhandled");
+ when(lambdaService.invokeArn(anyString(), any(), any())).thenReturn(errored);
+
+ FirehoseLambdaTransformer.Outcome outcome =
+ transform(stream(lambdaProcessor("NumberOfRetries", "2")), record("a"), record("b"));
+
+ assertTrue(outcome.records().isEmpty());
+ assertEquals(2, outcome.failedRecords());
+ verify(lambdaService, times(3)).invokeArn(anyString(), any(), any());
+ JsonNode line = errorLines().get(0);
+ assertEquals("Lambda.FunctionError", line.get("errorCode").asText());
+ assertEquals(3, line.get("attemptsMade").asInt());
+ }
+
+ @Test
+ void anUninvokableFunctionIsRetriedTheSameWay() throws Exception {
+ when(lambdaService.invokeArn(anyString(), any(), any()))
+ .thenThrow(new AwsException("ResourceNotFoundException", "Function not found", 404));
+
+ transform(stream(lambdaProcessor("NumberOfRetries", "1")), record("a"));
+
+ verify(lambdaService, times(2)).invokeArn(anyString(), any(), any());
+ assertEquals("Lambda.FunctionError", errorLines().get(0).get("errorCode").asText());
+ }
+
+ @Test
+ void aResponseCarryingNoRecordsIsNotRetried() throws Exception {
+ replyWith(ids -> "{}");
+
+ transform(stream(lambdaProcessor("NumberOfRetries", "5")), record("a"));
+
+ verify(lambdaService, times(1)).invokeArn(anyString(), any(), any());
+ JsonNode line = errorLines().get(0);
+ assertEquals("Lambda.MissingRecordId", line.get("errorCode").asText());
+ assertEquals(1, line.get("attemptsMade").asInt());
+ }
+
+ @Test
+ void anUnparseableResponseFailsEveryRecordWithoutRetrying() throws Exception {
+ replyWith(ids -> "not json at all");
+
+ transform(stream(lambdaProcessor()), record("a"));
+
+ verify(lambdaService, times(1)).invokeArn(anyString(), any(), any());
+ assertEquals("Lambda.MissingRecordId", errorLines().get(0).get("errorCode").asText());
+ }
+
+ @Test
+ void anAbsentNumberOfRetriesFallsBackToTheThreeAwsDefaults() {
+ InvokeResult errored = new InvokeResult();
+ errored.setFunctionError("Unhandled");
+ when(lambdaService.invokeArn(anyString(), any(), any())).thenReturn(errored);
+
+ transform(stream(lambdaProcessor()), record("a"));
+
+ verify(lambdaService, times(4)).invokeArn(anyString(), any(), any());
+ }
+
+ /**
+ * AWS stores a NumberOfRetries it never range-checks, so the largest one a caller can
+ * write has to stay a bounded number of invocations rather than overflowing the
+ * attempt count or occupying the flusher thread indefinitely.
+ */
+ @Test
+ void aRetryCountLargerThanFlociHonoursIsCapped() throws Exception {
+ InvokeResult errored = new InvokeResult();
+ errored.setFunctionError("Unhandled");
+ when(lambdaService.invokeArn(anyString(), any(), any())).thenReturn(errored);
+
+ transform(stream(lambdaProcessor("NumberOfRetries", String.valueOf(Integer.MAX_VALUE))), record("a"));
+
+ verify(lambdaService, times(101)).invokeArn(anyString(), any(), any());
+ assertEquals(101, errorLines().get(0).get("attemptsMade").asInt());
+ }
+
+ /**
+ * A NumberOfRetries too wide even for a long is still a number, and treating it as
+ * unparseable would hand it fewer attempts than a smaller one gets.
+ */
+ @Test
+ void aRetryCountTooWideForALongIsCappedRatherThanDefaulted() {
+ InvokeResult errored = new InvokeResult();
+ errored.setFunctionError("Unhandled");
+ when(lambdaService.invokeArn(anyString(), any(), any())).thenReturn(errored);
+
+ transform(stream(lambdaProcessor("NumberOfRetries", "999999999999999999999")), record("a"));
+
+ verify(lambdaService, times(101)).invokeArn(anyString(), any(), any());
+ }
+
+ @Test
+ void aNegativeRetryCountInvokesOnce() {
+ replyWith(ids -> records(ok(ids.get(0), "one")));
+
+ transform(stream(lambdaProcessor("NumberOfRetries", "-5")), record("a"));
+
+ verify(lambdaService, times(1)).invokeArn(anyString(), any(), any());
+ }
+
+ // ── error object ─────────────────────────────────────────────────────────
+
+ @Test
+ void theErrorObjectFollowsTheProbedMemberOrder() throws Exception {
+ replyWith(ids -> records(outcome(ids.get(0), "ProcessingFailed")));
+
+ transform(stream(lambdaProcessor()), record("a"));
+
+ JsonNode line = errorLines().get(0);
+ assertEquals(List.of("rawData", "errorCode", "errorMessage", "attemptsMade",
+ "arrivalTimestamp", "attemptEndingTimestamp", "lambdaARN"),
+ fieldNames(line));
+ assertEquals(FUNCTION_ARN, line.get("lambdaARN").asText());
+ assertEquals(DELIVERY_TIME.toEpochMilli(), line.get("arrivalTimestamp").asLong());
+ assertEquals(DELIVERY_TIME.toEpochMilli(), line.get("attemptEndingTimestamp").asLong());
+ }
+
+ @Test
+ void everyFailedRecordGetsItsOwnLineInOneErrorObject() throws Exception {
+ replyWith(ids -> records(outcome(ids.get(0), "ProcessingFailed"), outcome(ids.get(1), "ProcessingFailed")));
+
+ FirehoseLambdaTransformer.Outcome outcome =
+ transform(stream(lambdaProcessor()), record("a"), record("b"));
+
+ assertEquals(2, outcome.failedRecords());
+ List<JsonNode> lines = errorLines();
+ assertEquals(2, lines.size());
+ assertEquals(encode("a"), lines.get(0).get("rawData").asText());
+ assertEquals(encode("b"), lines.get(1).get("rawData").asText());
+ }
+
+ @Test
+ void theErrorOutputKeyResolvesTheErrorOutputTypeExpression() {
+ replyWith(ids -> records(outcome(ids.get(0), "ProcessingFailed")));
+
+ FirehoseLambdaTransformer.Outcome outcome = transform(stream(lambdaProcessor()), record("a"));
+
+ assertTrue(outcome.errorKey().startsWith("errors/processing-failed/stream-"),
+ "error output key was " + outcome.errorKey());
+ }
+
+ @Test
+ void theBatchIsWrittenToTheDestinationBucket() {
+ replyWith(ids -> records(outcome(ids.get(0), "ProcessingFailed")));
+
+ transform(stream(lambdaProcessor()), record("a"));
+
+ verify(s3Service).putObject(eq(BUCKET), anyString(), any(), eq("application/octet-stream"),
+ eq(Map.of()));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseProcessingConfigurationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseProcessingConfigurationTest.java
index 09f03d8c95..4c51294a87 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseProcessingConfigurationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseProcessingConfigurationTest.java
@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;
+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.BufferingHints;
import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;
import io.github.hectorvent.floci.services.kinesis.KinesisService;
import io.github.hectorvent.floci.services.s3.S3Service;
@@ -26,6 +27,7 @@ import java.util.function.Consumer;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@@ -70,7 +72,8 @@ class FirehoseProcessingConfigurationTest {
when(regionResolver.getAccountId()).thenReturn("000000000000");
service = new FirehoseService(storageFactory, mock(S3Service.class),
Mockito.mock(KinesisService.class), regionResolver, new MutableClock(), config,
- Mockito.mock(FirehoseParquetConverter.class));
+ Mockito.mock(FirehoseParquetConverter.class),
+ Mockito.mock(FirehoseLambdaTransformer.class));
}
private static ProcessorParameter parameter(String name, String value) {
@@ -94,6 +97,20 @@ class FirehoseProcessingConfigurationTest {
return processing("Lambda", parameters);
}
+ private static Processor lambdaProcessor(ProcessorParameter... parameters) {
+ Processor processor = new Processor();
+ processor.setType("Lambda");
+ processor.setParameters(new ArrayList<>(List.of(parameters)));
+ return processor;
+ }
+
+ private static BufferingHints hints(int sizeInMBs, int intervalInSeconds) {
+ BufferingHints hints = new BufferingHints();
+ hints.setSizeInMBs(sizeInMBs);
+ hints.setIntervalInSeconds(intervalInSeconds);
+ return hints;
+ }
+
private static S3Destination destination(Consumer<S3Destination> customizer) {
S3Destination s3 = new S3Destination();
s3.setRoleArn(ROLE_ARN);
@@ -329,7 +346,69 @@ class FirehoseProcessingConfigurationTest {
service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,
destination(s3 -> s3.setPrefix("changed/")));
- assertEquals(List.of("LambdaArn", "NumberOfRetries", "RoleArn"), storedParameterNames("stream"));
+ assertEquals(List.of("LambdaArn", "NumberOfRetries", "RoleArn", "BufferSizeInMBs",
+ "BufferIntervalInSeconds"), storedParameterNames("stream"));
+ }
+
+ /**
+ * Probed 2026-09-11: DescribeDeliveryStream echoes an omitted Enabled back as false
+ * here, where the conversion block echoes it as true, so the two cannot share a rule.
+ */
+ @Test
+ void anOmittedEnabledIsStoredAsDisabled() {
+ ProcessingConfiguration processing = new ProcessingConfiguration();
+ processing.setProcessors(List.of(lambdaProcessor(parameter("LambdaArn", LAMBDA_ARN))));
+ service.createDeliveryStream("stream",
+ destination(s3 -> s3.setProcessingConfiguration(processing)));
+
+ S3Destination stored = service.describeDeliveryStream("stream").s3Destination();
+ assertEquals(false, stored.getProcessingConfiguration().getEnabled());
+ assertFalse(stored.isProcessingEnabled());
+ }
+
+ /**
+ * The two buffer parameters are the Lambda processor's own defaults, not the
+ * destination's BufferingHints: probed, a destination buffering 5 MiB over 300s still
+ * echoes 1 and 60 on the processor.
+ */
+ @Test
+ void aProcessorWithoutBufferParametersGainsTheLambdaDefaults() {
+ service.createDeliveryStream("stream", destination(s3 -> {
+ s3.setBufferingHints(hints(5, 300));
+ s3.setProcessingConfiguration(lambdaProcessing(parameter("LambdaArn", LAMBDA_ARN)));
+ }));
+
+ List<ProcessorParameter> stored = service.describeDeliveryStream("stream").s3Destination()
+ .getProcessingConfiguration().getProcessors().get(0).getParameters();
+ assertEquals("1", parameterValue(stored, "BufferSizeInMBs"));
+ assertEquals("60", parameterValue(stored, "BufferIntervalInSeconds"));
+ }
+
+ /**
+ * State persisted before Enabled was defaulted never passes canonicalizeProcessors
+ * again, so the describe path heals it rather than reporting a stream without the
+ * member AWS always returns.
+ */
+ @Test
+ void applyDefaultsHealsAProcessingBlockStoredWithoutEnabled() {
+ ProcessingConfiguration processing = new ProcessingConfiguration();
+ processing.setProcessors(List.of(lambdaProcessor(parameter("LambdaArn", LAMBDA_ARN))));
+ S3Destination stored = new S3Destination();
+ stored.setProcessingConfiguration(processing);
+
+ stored.applyDefaults();
+
+ assertEquals(false, processing.getEnabled());
+ assertFalse(stored.isProcessingEnabled());
+ }
+
+ private static String parameterValue(List<ProcessorParameter> parameters, String name) {
+ for (ProcessorParameter parameter : parameters) {
+ if (name.equals(parameter.getParameterName())) {
+ return parameter.getParameterValue();
+ }
+ }
+ return null;
}
// Validation runs against the update's own content, since the update replaces the
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
index 9987fa65dc..97f895c3d5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
@@ -91,7 +91,8 @@ class FirehoseServiceTest {
// sees what a GetRecords consumer sees, and a stubbed one could not show that.
kinesisService = new KinesisService(storageFactory, regionResolver);
return new FirehoseService(storageFactory, s3Service, kinesisService,
- regionResolver, clock, config, mock(FirehoseParquetConverter.class));
+ regionResolver, clock, config, mock(FirehoseParquetConverter.class),
+ mock(FirehoseLambdaTransformer.class));
}
private static S3Destination destination(String bucketArn, String compressionFormat) {