Light, fluffy, and always free - The AWS Local Emulator alternative
feat(firehose): apply the Lambda transform to buffered records (#3393)
Kenshin Okinakad96516dc70eb
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseLambdaTransformDeliveryTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseLambdaTransformDeliveryTest.javanew file mode 100644index 0000000000..f76ce0756c--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseLambdaTransformDeliveryTest.java@@ -0,0 +1,251 @@+package com.floci.test;++import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.Assumptions;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;+import software.amazon.awssdk.core.SdkBytes;+import software.amazon.awssdk.services.firehose.FirehoseClient;+import software.amazon.awssdk.services.firehose.model.BufferingHints;+import software.amazon.awssdk.services.firehose.model.CompressionFormat;+import software.amazon.awssdk.services.firehose.model.CreateDeliveryStreamRequest;+import software.amazon.awssdk.services.firehose.model.DeleteDeliveryStreamRequest;+import software.amazon.awssdk.services.firehose.model.DeliveryStreamStatus;+import software.amazon.awssdk.services.firehose.model.DeliveryStreamType;+import software.amazon.awssdk.services.firehose.model.DescribeDeliveryStreamRequest;+import software.amazon.awssdk.services.firehose.model.ExtendedS3DestinationConfiguration;+import software.amazon.awssdk.services.firehose.model.ProcessingConfiguration;+import software.amazon.awssdk.services.firehose.model.Processor;+import software.amazon.awssdk.services.firehose.model.ProcessorParameter;+import software.amazon.awssdk.services.firehose.model.ProcessorParameterName;+import software.amazon.awssdk.services.firehose.model.ProcessorType;+import software.amazon.awssdk.services.firehose.model.PutRecordBatchRequest;+import software.amazon.awssdk.services.firehose.model.Record;+import software.amazon.awssdk.services.lambda.LambdaClient;+import software.amazon.awssdk.services.lambda.model.CreateFunctionRequest;+import software.amazon.awssdk.services.lambda.model.DeleteFunctionRequest;+import software.amazon.awssdk.services.lambda.model.FunctionCode;+import software.amazon.awssdk.services.lambda.model.Runtime;+import software.amazon.awssdk.services.s3.S3Client;+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;+import software.amazon.awssdk.services.s3.model.GetObjectRequest;+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;+import software.amazon.awssdk.services.s3.model.S3Object;++import java.nio.charset.StandardCharsets;+import java.util.Base64;+import java.util.List;+import java.util.Optional;++import static org.assertj.core.api.Assertions.assertThat;++/**+ * End-to-end proof that a transforming stream really invokes its function: the+ * records put through the SDK come back out of S3 uppercased by a Node.js+ * handler, the one it dropped is nowhere, and the one it failed lands under the+ * evaluated ErrorOutputPrefix as an AWS-shaped NDJSON error line.+ *+ * This is the only layer that runs the function for real. The emulator's own+ * tests mock the Lambda service, so the invocation payload, the base64 round+ * trip through the runtime, and the ARN resolution from the flusher thread are+ * exercised here and nowhere else.+ */+@DisplayName("Firehose record transformation delivery (Lambda)")+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class FirehoseLambdaTransformDeliveryTest {++ private static final int BUFFERING_INTERVAL_SECONDS = 5;+ private static final long POLL_TIMEOUT_MILLIS = 240_000L;+ private static final String KEPT_RECORD = "keep-me\n";+ private static final String DROPPED_RECORD = "DROP-me\n";+ private static final String FAILED_RECORD = "FAIL-me\n";++ private static FirehoseClient firehose;+ private static LambdaClient lambda;+ private static S3Client s3;+ private static String bucket;+ private static String streamName;+ private static String functionName;++ @BeforeAll+ static void setup() throws InterruptedException {+ String firehoseRole = TestFixtures.isRealAws()+ ? System.getenv("FIREHOSE_ROLE_ARN")+ : "arn:aws:iam::000000000000:role/firehose-role";+ String lambdaRole = TestFixtures.isRealAws()+ ? System.getenv("LAMBDA_ROLE_ARN")+ : "arn:aws:iam::000000000000:role/lambda-role";+ if (TestFixtures.isRealAws() && (firehoseRole == null || lambdaRole == null)) {+ Assumptions.abort("FIREHOSE_ROLE_ARN and LAMBDA_ROLE_ARN not set");+ }++ firehose = TestFixtures.firehoseClient();+ lambda = TestFixtures.lambdaClient();+ s3 = TestFixtures.s3Client();+ bucket = TestFixtures.uniqueName("firehose-transform");+ streamName = TestFixtures.uniqueName("sdk-transform");+ functionName = TestFixtures.uniqueName("sdk-transform-fn");++ s3.createBucket(CreateBucketRequest.builder().bucket(bucket).build());+ String functionArn = lambda.createFunction(CreateFunctionRequest.builder()+ .functionName(functionName)+ .runtime(Runtime.NODEJS20_X)+ .role(lambdaRole)+ .handler("index.handler")+ .timeout(30)+ .memorySize(256)+ .code(FunctionCode.builder()+ .zipFile(SdkBytes.fromByteArray(LambdaUtils.firehoseTransformZip()))+ .build())+ .build()).functionArn();++ firehose.createDeliveryStream(CreateDeliveryStreamRequest.builder()+ .deliveryStreamName(streamName)+ .deliveryStreamType(DeliveryStreamType.DIRECT_PUT)+ .extendedS3DestinationConfiguration(ExtendedS3DestinationConfiguration.builder()+ .roleARN(firehoseRole)+ .bucketARN("arn:aws:s3:::" + bucket)+ .prefix("data/")+ .errorOutputPrefix("errors/!{firehose:error-output-type}/")+ .compressionFormat(CompressionFormat.UNCOMPRESSED)+ .bufferingHints(BufferingHints.builder()+ .sizeInMBs(5)+ .intervalInSeconds(BUFFERING_INTERVAL_SECONDS)+ .build())+ .processingConfiguration(ProcessingConfiguration.builder()+ .enabled(true)+ .processors(Processor.builder()+ .type(ProcessorType.LAMBDA)+ .parameters(ProcessorParameter.builder()+ .parameterName(ProcessorParameterName.LAMBDA_ARN)+ .parameterValue(functionArn)+ .build())+ .build())+ .build())+ .build())+ .build());++ awaitStreamActive();++ assertThat(firehose.putRecordBatch(PutRecordBatchRequest.builder()+ .deliveryStreamName(streamName)+ .records(record(KEPT_RECORD), record(DROPPED_RECORD), record(FAILED_RECORD))+ .build()).failedPutCount()).isZero();+ }++ @Test+ @Order(1)+ @DisplayName("the delivered object holds what the function returned, and nothing it dropped")+ void deliversTheTransformedRecord() throws InterruptedException {+ String delivered = new String(awaitObject("data/"), StandardCharsets.UTF_8);++ assertThat(delivered).contains("KEEP-ME");+ assertThat(delivered).doesNotContain("keep-me");+ assertThat(delivered).doesNotContain("DROP-me").doesNotContain("DROP-ME");+ assertThat(delivered).doesNotContain("FAIL-me").doesNotContain("FAIL-ME");+ }++ @Test+ @Order(2)+ @DisplayName("a ProcessingFailed record lands under the processing-failed error output")+ void writesTheFailedRecordToTheErrorOutput() throws InterruptedException {+ String errorLine = new String(awaitObject("errors/processing-failed/"), StandardCharsets.UTF_8).strip();++ assertThat(errorLine).contains("\"errorCode\":\"Lambda.ProcessingFailedStatus\"");+ assertThat(errorLine).contains("\"lambdaARN\":\"");+ assertThat(errorLine).contains("\"rawData\":\""+ + Base64.getEncoder().encodeToString(FAILED_RECORD.getBytes(StandardCharsets.UTF_8)) + "\"");+ }++ /**+ * The buffer has to flush and the function has to run, which on a cold container is+ * well past the buffering interval, so both tests poll rather than sleep once.+ */+ private static byte[] awaitObject(String prefix) throws InterruptedException {+ long deadline = System.currentTimeMillis() + POLL_TIMEOUT_MILLIS;+ while (true) {+ Optional<S3Object> found = firstObject(prefix);+ if (found.isPresent()) {+ return s3.getObjectAsBytes(GetObjectRequest.builder()+ .bucket(bucket).key(found.get().key()).build()).asByteArray();+ }+ if (System.currentTimeMillis() >= deadline) {+ throw new AssertionError("no object under " + prefix + " in " + bucket + " after "+ + POLL_TIMEOUT_MILLIS / 1000 + "s");+ }+ Thread.sleep(5_000L);+ }+ }++ private static Optional<S3Object> firstObject(String prefix) {+ List<S3Object> objects = s3.listObjectsV2(ListObjectsV2Request.builder()+ .bucket(bucket).prefix(prefix).build()).contents();+ return objects.isEmpty() ? Optional.empty() : Optional.of(objects.get(0));+ }++ // Real AWS returns from CreateDeliveryStream while the stream is still CREATING+ // and rejects PutRecordBatch until it turns ACTIVE; Floci creates it active, so+ // this returns on the first poll locally.+ private static void awaitStreamActive() throws InterruptedException {+ long deadline = System.currentTimeMillis() + POLL_TIMEOUT_MILLIS;+ while (true) {+ DeliveryStreamStatus status = firehose.describeDeliveryStream(+ DescribeDeliveryStreamRequest.builder()+ .deliveryStreamName(streamName).build())+ .deliveryStreamDescription().deliveryStreamStatus();+ if (status == DeliveryStreamStatus.ACTIVE) {+ return;+ }+ if (System.currentTimeMillis() >= deadline) {+ throw new AssertionError("delivery stream stayed " + status + " for "+ + POLL_TIMEOUT_MILLIS / 1000 + "s");+ }+ Thread.sleep(5_000L);+ }+ }++ private static Record record(String data) {+ return Record.builder().data(SdkBytes.fromString(data, StandardCharsets.UTF_8)).build();+ }++ @AfterAll+ static void cleanup() {+ if (firehose != null) {+ try {+ firehose.deleteDeliveryStream(DeleteDeliveryStreamRequest.builder()+ .deliveryStreamName(streamName).build());+ } catch (Exception e) {+ // Best effort: a stream that never got created must not fail the run.+ System.out.println("Cleanup of delivery stream " + streamName + " failed: " + e);+ }+ firehose.close();+ }+ if (lambda != null) {+ try {+ lambda.deleteFunction(DeleteFunctionRequest.builder()+ .functionName(functionName).build());+ } catch (Exception e) {+ // Best effort cleanup of the transform function.+ System.out.println("Cleanup of function " + functionName + " failed: " + e);+ }+ lambda.close();+ }+ if (s3 != null) {+ try {+ s3.listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()).contents()+ .forEach(object -> s3.deleteObject(DeleteObjectRequest.builder()+ .bucket(bucket).key(object.key()).build()));+ s3.deleteBucket(builder -> builder.bucket(bucket));+ } catch (Exception e) {+ // Best effort cleanup of the probe bucket.+ System.out.println("Cleanup of bucket " + bucket + " failed: " + e);+ }+ s3.close();+ }+ }+}diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.javaindex d559780c70..fa7de92808 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java@@ -32,6 +32,33 @@ public final class LambdaUtils {return createZip("index.js", code);}+ /**+ * ZIP containing a Node.js handler shaped like a Firehose transform: it uppercases each+ * record, drops one whose payload says DROP, and reports one whose payload says FAIL,+ * so a single invocation exercises all three result values.+ */+ public static byte[] firehoseTransformZip() {+ String code = """+ exports.handler = async (event) => ({+ records: (event.records || []).map((r) => {+ const data = Buffer.from(r.data, 'base64').toString('utf8');+ if (data.includes('DROP')) {+ return { recordId: r.recordId, result: 'Dropped' };+ }+ if (data.includes('FAIL')) {+ return { recordId: r.recordId, result: 'ProcessingFailed' };+ }+ return {+ recordId: r.recordId,+ result: 'Ok',+ data: Buffer.from(data.toUpperCase(), 'utf8').toString('base64')+ };+ })+ });+ """;+ return createZip("index.js", code);+ }+/*** ZIP containing a Ruby handler that greets by name.*/diff --git a/docs/services/firehose.md b/docs/services/firehose.mdindex 1d579f5c7f..500e59f590 100644--- a/docs/services/firehose.md+++ b/docs/services/firehose.md@@ -85,15 +85,19 @@ An omitted `Enabled` counts as enabled; `Enabled: false` stores and merges the c## Record transformation-`ProcessingConfiguration` on the extended S3 destination is modelled, validated and echoed, but **not applied**: a stream that enables a transformation is accepted and delivers its records untransformed, and create and update log a warning saying so. Validation follows AWS, down to the messages and the order they are reported in.+`ProcessingConfiguration` on the extended S3 destination is applied at delivery: a flushed buffer goes to the Lambda processor's function as one invocation, `Ok` records deliver the returned data, and `Dropped` records leave no trace. Any other per-record outcome goes to the error output while the rest of the batch still delivers, one NDJSON line each under the `ErrorOutputPrefix` with `!{firehose:error-output-type}` resolving to `processing-failed`.-Two behaviors are worth knowing because they differ from the conversion block above. `UpdateDestination` replaces this block whole rather than merging it member-wise, so an update carrying only `{"Enabled": false}` leaves no processors behind, and each update is validated against its own content rather than the merged result. And a stored Lambda processor is echoed with `NumberOfRetries` and `RoleArn` filled in and its parameters reordered, as AWS returns them.+A function that errors is retried `NumberOfRetries` more times and then fails the whole batch as `Lambda.FunctionError`; a response the function returned successfully is final however malformed it is. The transformation runs before data format conversion, so a stream configured for both converts what the function returned.++Two behaviors invert what the conversion block above does, both probed rather than assumed: `UpdateDestination` replaces this block whole instead of merging it member-wise, so an update carrying only `{"Enabled": false}` leaves no processors behind and is validated against its own content; and an omitted `Enabled` leaves the transformation **disabled**, where on the conversion block it means enabled. A stored Lambda processor is echoed with `NumberOfRetries`, `RoleArn`, `BufferSizeInMBs` and `BufferIntervalInSeconds` filled in and reordered, as AWS returns them; the two buffer values default to 1 and 60, the processor's own defaults rather than anything `BufferingHints` says.### Known deviations from AWS-- The transformation is not applied. A transform function is never invoked, so records reach the destination exactly as they were put, which is what makes a local test of one pass where AWS would not.-- Two checks AWS itself does not make are deliberately absent: `NumberOfRetries` is not range-checked, despite the documented 1 to 8, and the function a `LambdaArn` names is not required to exist.-- A null entry in `Processors` or `Parameters`, which a raw JSON client can send, is ignored rather than reported. Real AWS answers `InternalFailure` there, a fault of its own that is not worth reproducing; skipping the entry leaves the same configuration as omitting it would.+- Only the `Lambda` processor type is applied; the others are stored and echoed.+- The whole buffer is one invocation, so the processor's buffering parameters do not re-buffer and no payload size limit is enforced.+- `recordId` is a UUID rather than AWS's longer sequence token, and every timestamp is the delivery time: Floci tracks no per-record arrival.+- Neither `NumberOfRetries` nor the existence of the function a `LambdaArn` names is checked at configuration time, as on AWS. At delivery an ARN naming nothing fails as `Lambda.FunctionError`, and a `NumberOfRetries` above 100 is capped, a bound AWS does not impose so that one flush cannot occupy the flusher indefinitely.+- A null entry in `Processors` or `Parameters`, which a raw JSON client can send, is ignored rather than reported; real AWS answers `InternalFailure` there, a fault of its own not worth reproducing.## S3 object keysdiff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformer.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformer.javanew file mode 100644index 0000000000..498a84a951--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformer.java@@ -0,0 +1,388 @@+package io.github.hectorvent.floci.services.firehose;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;+import io.github.hectorvent.floci.services.lambda.LambdaService;+import io.github.hectorvent.floci.services.lambda.model.InvocationType;+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;+import io.github.hectorvent.floci.services.s3.S3Service;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.io.ByteArrayOutputStream;+import java.math.BigInteger;+import java.time.Instant;+import java.util.ArrayList;+import java.util.Base64;+import java.util.HashSet;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.UUID;++/**+ * Runs a flushed batch through the stream's Lambda transform before it is+ * delivered: one invocation per batch carrying every buffered record, the+ * returned data replacing the original, and records the function dropped or+ * failed taken out of the delivery. The failed ones go to the error output,+ * one NDJSON line each, under the evaluated ErrorOutputPrefix with+ * {@code !{firehose:error-output-type}} resolved to {@code processing-failed}.+ *+ * The per-record semantics were probed against real AWS (us-west-2,+ * 2026-09-10): {@code Ok} delivers the returned data, {@code Dropped} leaves no+ * trace at all, and anything else routes that record to the error output while+ * the rest of the batch still delivers. A function that errors is retried, a+ * response the function shaped wrongly is not. Every message below is AWS's own+ * except the two {@code Lambda.InvalidReturnFormat} ones this class invents, for+ * a {@code result} AWS does not define and for {@code data} that is not base64.+ *+ * This runs ahead of data format conversion, so a stream with both configured+ * converts what the transform returned.+ */+@ApplicationScoped+public class FirehoseLambdaTransformer {++ private static final Logger LOG = Logger.getLogger(FirehoseLambdaTransformer.class);+ private static final String ERROR_OUTPUT_TYPE = "processing-failed";+ private static final String CONTENT_TYPE = "application/octet-stream";+ private static final int DEFAULT_NUMBER_OF_RETRIES = 3;+ private static final int MAX_NUMBER_OF_RETRIES = 100;+ private static final String RESULT_OK = "Ok";+ private static final String RESULT_DROPPED = "Dropped";+ private static final String RESULT_PROCESSING_FAILED = "ProcessingFailed";++ private static final String FUNCTION_ERROR_MESSAGE =+ "The Lambda function was successfully invoked but it returned an error result.";+ private static final String MISSING_RECORD_ID_MESSAGE =+ "One or more record Ids were not returned. Ensure that the Lambda function returns"+ + " all received record Ids.";+ private static final String DUPLICATED_RECORD_ID_MESSAGE =+ "Multiple records were returned with the same record Id. Ensure that the Lambda"+ + " function returns a unique record Id for each record.";+ private static final String PROCESSING_FAILED_MESSAGE = "ProcessingFailed status set for record";+ private static final String NULL_DATA_MESSAGE =+ "The data field cannot be null when the SourceType is DirectPut or KinesisStreamAsSource"+ + " and processing result is Ok. The value field cannot be null when the SourceType"+ + " is MSKAsSource and processing result is Ok.";++ private final LambdaService lambdaService;+ private final S3Service s3Service;+ private final ObjectMapper mapper;+ private final RegionResolver regionResolver;++ @Inject+ public FirehoseLambdaTransformer(LambdaService lambdaService, S3Service s3Service,+ ObjectMapper mapper, RegionResolver regionResolver) {+ this.lambdaService = lambdaService;+ this.s3Service = s3Service;+ this.mapper = mapper;+ this.regionResolver = regionResolver;+ }++ /** What a transformation produced: the records still to deliver, plus the counts for the flush log line. */+ public record Outcome(List<byte[]> records, int droppedRecords, int failedRecords, String errorKey) {}++ public Outcome transform(DeliveryStreamDescription stream, String bucket, List<byte[]> records,+ Instant deliveryTime) {+ S3Destination s3 = stream.s3Destination();+ Processor processor = lambdaProcessor(s3);+ if (processor == null) {+ return new Outcome(records, 0, 0, null);+ }+ String functionArn = parameterValue(processor, "LambdaArn");++ List<String> recordIds = new ArrayList<>(records.size());+ for (int i = 0; i < records.size(); i++) {+ recordIds.add(UUID.randomUUID().toString());+ }++ Invocation invocation = invoke(stream, functionArn, recordIds, records, deliveryTime,+ numberOfRetries(processor) + 1);++ List<byte[]> delivered = new ArrayList<>(records.size());+ List<FailedRecord> failures = new ArrayList<>();+ int dropped = 0;+ for (int i = 0; i < records.size(); i++) {+ byte[] record = records.get(i);+ Verdict verdict = invocation.verdictFor(recordIds.get(i));+ if (verdict.errorCode() != null) {+ failures.add(new FailedRecord(record, verdict.errorCode(), verdict.errorMessage()));+ } else if (verdict.data() == null) {+ dropped++;+ } else {+ delivered.add(verdict.data());+ }+ }++ String errorKey = failures.isEmpty() ? null+ : writeErrorOutput(stream, s3, bucket, failures, deliveryTime, functionArn,+ invocation.attemptsMade());+ return new Outcome(delivered, dropped, failures.size(), errorKey);+ }++ // ── configuration ────────────────────────────────────────────────────────++ /**+ * The single Lambda processor of an enabled ProcessingConfiguration, or null when the+ * stream has none: the other processor types AWS models are stored and echoed but not+ * applied. The validator has already made "at most one" true.+ */+ private static Processor lambdaProcessor(S3Destination s3) {+ if (s3 == null || !s3.isProcessingEnabled() || s3.getProcessingConfiguration().getProcessors() == null) {+ return null;+ }+ for (Processor processor : s3.getProcessingConfiguration().getProcessors()) {+ if (processor != null && "Lambda".equals(processor.getType())+ && parameterValue(processor, "LambdaArn") != null) {+ return processor;+ }+ }+ return null;+ }++ /**+ * AWS does not range-check NumberOfRetries at configuration time, so a stored value+ * can be any number the parameter's string type allows, 2147483647 included. It is+ * clamped rather than honored: a flush occupies the single flusher thread, so a retry+ * count that large would stall every other stream's delivery behind it. The cap is+ * well above the documented 1 to 8 and the 9 the probe saw honored. A value that is+ * not a number falls back to the 3 AWS itself defaults to.+ *+ * Read as a BigInteger so the clamp holds at any magnitude: a value too wide even for+ * a long is still a number, and treating it as unparseable would hand a larger+ * NumberOfRetries fewer attempts than a smaller one.+ */+ private static int numberOfRetries(Processor processor) {+ String configured = parameterValue(processor, "NumberOfRetries");+ if (configured == null) {+ return DEFAULT_NUMBER_OF_RETRIES;+ }+ try {+ return new BigInteger(configured.trim())+ .max(BigInteger.ZERO)+ .min(BigInteger.valueOf(MAX_NUMBER_OF_RETRIES))+ .intValue();+ } catch (NumberFormatException e) {+ LOG.warnv("Firehose Lambda processor has a non-numeric NumberOfRetries {0}, using {1}",+ configured, DEFAULT_NUMBER_OF_RETRIES);+ return DEFAULT_NUMBER_OF_RETRIES;+ }+ }++ private static String parameterValue(Processor processor, String name) {+ if (processor.getParameters() == null) {+ return null;+ }+ for (ProcessorParameter parameter : processor.getParameters()) {+ if (parameter != null && name.equals(parameter.getParameterName())) {+ return parameter.getParameterValue();+ }+ }+ return null;+ }++ // ── invocation ───────────────────────────────────────────────────────────++ /**+ * What one batch's invocation settled on: either a function-level failure every+ * record inherits, or the per-record results the function returned.+ */+ private record Invocation(int attemptsMade, boolean functionError,+ Map<String, LambdaResult> results, Set<String> duplicatedIds) {++ Verdict verdictFor(String recordId) {+ if (functionError) {+ return Verdict.failed("Lambda.FunctionError", FUNCTION_ERROR_MESSAGE);+ }+ if (duplicatedIds.contains(recordId)) {+ return Verdict.failed("Lambda.DuplicatedRecordId", DUPLICATED_RECORD_ID_MESSAGE);+ }+ LambdaResult result = results.get(recordId);+ if (result == null) {+ return Verdict.failed("Lambda.MissingRecordId", MISSING_RECORD_ID_MESSAGE);+ }+ if (RESULT_DROPPED.equals(result.result())) {+ return Verdict.dropped();+ }+ if (RESULT_PROCESSING_FAILED.equals(result.result())) {+ return Verdict.failed("Lambda.ProcessingFailedStatus", PROCESSING_FAILED_MESSAGE);+ }+ if (!RESULT_OK.equals(result.result())) {+ return Verdict.failed("Lambda.InvalidReturnFormat",+ "The result field must be one of Ok, Dropped or ProcessingFailed.");+ }+ if (result.data() == null) {+ return Verdict.failed("Lambda.InvalidReturnFormat", NULL_DATA_MESSAGE);+ }+ try {+ return Verdict.transformed(Base64.getDecoder().decode(result.data()));+ } catch (IllegalArgumentException e) {+ return Verdict.failed("Lambda.InvalidReturnFormat",+ "The data field must be base64 encoded: " + e.getMessage());+ }+ }+ }++ private record LambdaResult(String result, String data) {}++ /** A transformed payload, a failure, or, with every member null, a drop. */+ private record Verdict(byte[] data, String errorCode, String errorMessage) {++ static Verdict transformed(byte[] data) {+ return new Verdict(data, null, null);+ }++ static Verdict dropped() {+ return new Verdict(null, null, null);+ }++ static Verdict failed(String errorCode, String errorMessage) {+ return new Verdict(null, errorCode, errorMessage);+ }+ }++ /**+ * Invokes the function, retrying only a function-level error: a response the+ * function returned successfully is taken as final however malformed it is (probed,+ * a response with no records at all reports attemptsMade 1 while a throwing+ * function reports NumberOfRetries + 1).+ */+ private Invocation invoke(DeliveryStreamDescription stream, String functionArn, List<String> recordIds,+ List<byte[]> records, Instant deliveryTime, int maxAttempts) {+ byte[] payload = buildPayload(stream, recordIds, records, deliveryTime);+ for (int attempt = 1; attempt <= maxAttempts; attempt++) {+ try {+ InvokeResult result = lambdaService.invokeArn(functionArn, payload, InvocationType.RequestResponse);+ if (result.getFunctionError() == null) {+ return parseResponse(attempt, result.getPayload());+ }+ LOG.warnv("Firehose transform {0} returned {1} on attempt {2} of {3}",+ functionArn, result.getFunctionError(), attempt, maxAttempts);+ } catch (Exception e) {+ LOG.warnv("Firehose transform {0} could not be invoked on attempt {1} of {2}: {3}",+ functionArn, attempt, maxAttempts, e.getMessage());+ }+ }+ return new Invocation(maxAttempts, true, Map.of(), Set.of());+ }++ private byte[] buildPayload(DeliveryStreamDescription stream, List<String> recordIds,+ List<byte[]> records, Instant deliveryTime) {+ Map<String, Object> event = new LinkedHashMap<>();+ event.put("invocationId", UUID.randomUUID().toString());+ event.put("deliveryStreamArn", stream.getDeliveryStreamARN());+ event.put("region", AwsArnUtils.regionOrDefault(stream.getDeliveryStreamARN(),+ regionResolver.getDefaultRegion()));+ List<Map<String, Object>> eventRecords = new ArrayList<>(records.size());+ for (int i = 0; i < records.size(); i++) {+ Map<String, Object> eventRecord = new LinkedHashMap<>();+ eventRecord.put("recordId", recordIds.get(i));+ // Floci does not track per-record arrival, so the batch's delivery time stands+ // in for all of them, as it does in the error output.+ eventRecord.put("approximateArrivalTimestamp", deliveryTime.toEpochMilli());+ eventRecord.put("data", Base64.getEncoder().encodeToString(records.get(i)));+ eventRecords.add(eventRecord);+ }+ event.put("records", eventRecords);+ try {+ return mapper.writeValueAsBytes(event);+ } catch (Exception e) {+ throw new AwsException("InternalServerException",+ "Failed to serialize a Firehose transformation payload: " + e.getMessage(), 500);+ }+ }++ /**+ * A response that carries no usable records array leaves every record unaccounted+ * for, which is what AWS reports for it: Lambda.MissingRecordId per record rather+ * than a batch-level failure (probed). An entry naming a record the batch never+ * contained is ignored, also as AWS does.+ */+ private Invocation parseResponse(int attempt, byte[] responsePayload) {+ Map<String, LambdaResult> results = new LinkedHashMap<>();+ Set<String> duplicatedIds = new HashSet<>();+ JsonNode records = readRecords(responsePayload);+ if (records != null) {+ for (JsonNode record : records) {+ String recordId = text(record, "recordId");+ if (recordId == null) {+ continue;+ }+ LambdaResult result = new LambdaResult(text(record, "result"), text(record, "data"));+ if (results.putIfAbsent(recordId, result) != null) {+ duplicatedIds.add(recordId);+ }+ }+ }+ return new Invocation(attempt, false, results, duplicatedIds);+ }++ private JsonNode readRecords(byte[] responsePayload) {+ if (responsePayload == null || responsePayload.length == 0) {+ return null;+ }+ try {+ JsonNode response = mapper.readTree(responsePayload);+ JsonNode records = response == null ? null : response.get("records");+ return records != null && records.isArray() ? records : null;+ } catch (Exception e) {+ LOG.warnv("Firehose transform returned a response that is not JSON: {0}", e.getMessage());+ return null;+ }+ }++ private static String text(JsonNode node, String field) {+ JsonNode value = node.get(field);+ return value == null || !value.isTextual() ? null : value.asText();+ }++ // ── error output ─────────────────────────────────────────────────────────++ private record FailedRecord(byte[] data, String errorCode, String errorMessage) {}++ /**+ * One NDJSON line per failed record, in the shape and member order real AWS writes+ * (probed). It is not the format-conversion error object: that one leads with+ * attemptsMade and carries lastErrorCode and a dataCatalogTable block, where this+ * one leads with rawData and ends with lambdaARN, in that casing.+ *+ * Both timestamps are the delivery time, a deviation: Floci tracks no per-record+ * arrival. attemptsMade counts the batch's attempts, a record never having retries+ * of its own.+ */+ private String writeErrorOutput(DeliveryStreamDescription stream, S3Destination s3, String bucket,+ List<FailedRecord> failures, Instant deliveryTime,+ String functionArn, int attemptsMade) {+ String errorKey = S3ObjectKeyResolver.resolveErrorKey(s3, stream.getDeliveryStreamName(),+ stream.getVersionId(), deliveryTime, ERROR_OUTPUT_TYPE);+ ByteArrayOutputStream body = new ByteArrayOutputStream();+ for (FailedRecord failure : failures) {+ Map<String, Object> line = new LinkedHashMap<>();+ line.put("rawData", Base64.getEncoder().encodeToString(failure.data()));+ line.put("errorCode", failure.errorCode());+ line.put("errorMessage", failure.errorMessage());+ line.put("attemptsMade", attemptsMade);+ line.put("arrivalTimestamp", deliveryTime.toEpochMilli());+ line.put("attemptEndingTimestamp", deliveryTime.toEpochMilli());+ line.put("lambdaARN", functionArn);+ try {+ body.write(mapper.writeValueAsBytes(line));+ body.write('\n');+ } catch (Exception e) {+ throw new AwsException("InternalServerException",+ "Failed to serialize a Firehose error-output record: " + e.getMessage(), 500);+ }+ }+ s3Service.putObject(bucket, errorKey, body.toByteArray(), CONTENT_TYPE, Map.of());+ return errorKey;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.javaindex 0587e25e24..86990d586c 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java@@ -102,6 +102,7 @@ public class FirehoseService implements ResourceProvider {// commits exactly the position that snapshot covered.private final Map<String, Map<String, String>> pendingSourceIterators = new ConcurrentHashMap<>();private final FirehoseParquetConverter parquetConverter;+ private final FirehoseLambdaTransformer lambdaTransformer;private final Clock clock;private final long tickIntervalSeconds;private final int flushRecordCount;@@ -187,7 +188,8 @@ public class FirehoseService implements ResourceProvider {@Injectpublic FirehoseService(StorageFactory storageFactory, S3Service s3Service, KinesisService kinesisService,RegionResolver regionResolver, Clock clock, EmulatorConfig config,- FirehoseParquetConverter parquetConverter) {+ FirehoseParquetConverter parquetConverter,+ FirehoseLambdaTransformer lambdaTransformer) {this.streamStore = storageFactory.create("firehose", "streams.json",new TypeReference<Map<String, DeliveryStreamDescription>>() {});this.sourceIteratorStore = storageFactory.create("firehose", "source-iterators.json",@@ -196,6 +198,7 @@ public class FirehoseService implements ResourceProvider {this.kinesisService = kinesisService;this.regionResolver = regionResolver;this.parquetConverter = parquetConverter;+ this.lambdaTransformer = lambdaTransformer;this.clock = clock;this.tickIntervalSeconds = Math.max(1, config.services().firehose().tickIntervalSeconds());this.flushRecordCount = Math.max(0, config.services().firehose().flushRecordCount());@@ -313,7 +316,6 @@ public class FirehoseService implements ResourceProvider {if (s3Config != null) {s3Config.canonicalizeProcessors();}- warnIfProcessingEnabled(name, s3Config);String arn = AwsArnUtils.Arn.of("firehose", region, accountId,"deliverystream/" + name).toString();// CreateDeliveryStream's KinesisStreamSourceConfiguration carries only the ARN and@@ -392,7 +394,6 @@ public class FirehoseService implements ResourceProvider {stream.setLastUpdateTimestamp(java.time.Instant.now());streamPut(streamKey, stream);LOG.infov("Updated destination {0} of Firehose delivery stream {1}", destinationId, name);- warnIfProcessingEnabled(name, stream.s3Destination());}public void startDeliveryStreamEncryption(String name, String keyType, String keyArn) {@@ -464,20 +465,6 @@ public class FirehoseService implements ResourceProvider {}}- /**- * Says, where the configuration is set, that the transformation will not be applied.- * Fires on create and on every update that leaves it enabled, so a caller who keeps- * changing the destination keeps being told. Deliberately not in the flush path,- * which runs on every buffered delivery: create and update are caller-driven and are- * the moments a caller can act on the warning.- */- private static void warnIfProcessingEnabled(String name, S3Destination s3) {- if (s3 != null && s3.isProcessingEnabled()) {- LOG.warnv("Delivery stream {0} enables a record transformation, which Floci does not"- + " apply yet; its records will be delivered untransformed", name);- }- }-private static void mergeDestination(S3Destination current, S3Destination update) {if (update.getRoleArn() != null) current.setRoleArn(update.getRoleArn());if (update.getBucketArn() != null) current.setBucketArn(update.getBucketArn());@@ -893,10 +880,26 @@ public class FirehoseService implements ResourceProvider {try {String bucket = resolveBucket(stream);S3Destination s3 = stream.s3Destination();+ List<byte[]> records = toFlush;+ if (s3 != null && s3.isProcessingEnabled()) {+ ensureBucket(bucket);+ FirehoseLambdaTransformer.Outcome transformed =+ lambdaTransformer.transform(stream, bucket, records, clock.instant());+ LOG.infov("Transformed {0} records from stream {1} ({2} dropped, {3} failed)",+ records.size(), streamName, transformed.droppedRecords(), transformed.failedRecords());+ records = transformed.records();+ if (records.isEmpty()) {+ // Nothing survived the transform. The batch is accounted for, dropped+ // records deliberately leaving no trace and failed ones already in the+ // error output, so the source may advance past it.+ commitSourceIterators(streamName, checkpoint);+ return;+ }+ }if (s3 != null && s3.isDataFormatConversionEnabled()) {ensureBucket(bucket);FirehoseParquetConverter.Outcome outcome =- parquetConverter.deliver(stream, bucket, toFlush, clock.instant());+ parquetConverter.deliver(stream, bucket, records, clock.instant());LOG.infov("Converted {0} records ({1} failed) from stream {2} to s3://{3}/{4}",outcome.convertedRecords(), outcome.failedRecords(), streamName, bucket,outcome.dataKey() != null ? outcome.dataKey() : outcome.errorKey());@@ -917,7 +920,7 @@ public class FirehoseService implements ResourceProvider {// (verified: three "abc" records arrive as the 9 bytes "abcabcabc").// See the deviation noted in docs/services/firehose.md.ByteArrayOutputStream payload = new ByteArrayOutputStream();- for (byte[] data : toFlush) {+ for (byte[] data : records) {payload.writeBytes(data);if (data.length > 0 && data[data.length - 1] != '\n') {payload.write('\n');@@ -928,7 +931,7 @@ public class FirehoseService implements ResourceProvider {s3Service.putObject(bucket, key, body, "application/octet-stream", Map.of(),new PutObjectOptions().withContentEncoding(compression.contentEncoding()));LOG.infov("Flushed {0} records from stream {1} to s3://{2}/{3} ({4})",- toFlush.size(), streamName, bucket, key, compression.wireValue());+ records.size(), streamName, bucket, key, compression.wireValue());// Only now: the records these iterators were read past are durable.commitSourceIterators(streamName, checkpoint);} catch (Exception e) {diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.javaindex 163982a4e0..5d8866ca76 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java@@ -263,17 +263,18 @@ public class DeliveryStreamDescription {}/**- * True when a transformation applies to deliveries. An omitted Enabled is taken- * as enabled, following the conversion block, where that was probed; the- * processing block's own behavior for an omitted Enabled was not, and nothing- * but the "not applied yet" warning reads this today.+ * True when a transformation applies to deliveries: the configuration is present+ * and Enabled is explicitly true. An omitted Enabled leaves it disabled, the+ * opposite of the conversion block below. Probed rather than carried over from+ * there (2026-09-11): DescribeDeliveryStream echoes an omitted Enabled as false+ * here and as true on the conversion block.** Ignored for serialization for the same reason as the accessor below.*/@JsonIgnorepublic boolean isProcessingEnabled() {return processingConfiguration != null- && !Boolean.FALSE.equals(processingConfiguration.getEnabled());+ && Boolean.TRUE.equals(processingConfiguration.getEnabled());}/**@@ -326,13 +327,25 @@ public class DeliveryStreamDescription {bufferingHints.setIntervalInSeconds(BufferingHints.DEFAULT_INTERVAL_SECONDS);}}+ // Legacy persisted state again: a processing block stored before Enabled was+ // defaulted never passes canonicalizeProcessors a second time, and would be+ // described without the member AWS always returns. Only Enabled is healed+ // here, since the parameters AWS defaults include RoleArn, which a read must+ // not inject from whatever role is current by then.+ if (processingConfiguration != null && processingConfiguration.getEnabled() == null) {+ processingConfiguration.setEnabled(false);+ }}/*** Real AWS answers with more than the caller sent: a Lambda processor comes back- * with NumberOfRetries defaulted to 3 and RoleArn filled from the delivery- * stream's role, and the parameters are echoed in a fixed order whatever order- * they arrived in (probed 2026-09-10).+ * with NumberOfRetries defaulted to 3, RoleArn filled from the delivery stream's+ * role, and BufferSizeInMBs and BufferIntervalInSeconds defaulted to 1 and 60,+ * and the parameters are echoed in a fixed order whatever order they arrived in+ * (probed 2026-09-10 and 2026-09-11). The two buffer defaults are the Lambda+ * processor's own, not the destination's BufferingHints: a destination buffering+ * 5 MiB over 300s still echoes 1 and 60 here. An omitted Enabled is stored as+ * false, which is what leaves the transformation disabled.** Called as a destination is written, not as it is read. Doing it on the way out* would mutate what storage handed back, so whether a Describe had happened would@@ -340,7 +353,13 @@ public class DeliveryStreamDescription {* role was current at the time of the read.*/public void canonicalizeProcessors() {- if (processingConfiguration == null || processingConfiguration.getProcessors() == null) {+ if (processingConfiguration == null) {+ return;+ }+ if (processingConfiguration.getEnabled() == null) {+ processingConfiguration.setEnabled(false);+ }+ if (processingConfiguration.getProcessors() == null) {return;}for (Processor processor : processingConfiguration.getProcessors()) {@@ -360,6 +379,8 @@ public class DeliveryStreamDescription {if (roleArn != null) {byName.putIfAbsent("RoleArn", roleArn);}+ byName.putIfAbsent("BufferSizeInMBs", "1");+ byName.putIfAbsent("BufferIntervalInSeconds", "60");List<ProcessorParameter> ordered = new ArrayList<>(byName.size());for (String name : LAMBDA_PARAMETER_ORDER) {if (byName.containsKey(name)) {diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionTest.javaindex 9f9d733273..a6cf8fd15f 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseDataFormatConversionTest.java@@ -15,6 +15,9 @@ import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescriptimport io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OrcSerDe;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.OutputFormatConfiguration;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ParquetSerDe;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.SchemaConfiguration;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Serializer;@@ -23,6 +26,7 @@ import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.testing.MutableClock;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.mockito.ArgumentCaptor;import org.mockito.Mockito;import java.util.HashMap;@@ -59,6 +63,7 @@ class FirehoseDataFormatConversionTest {private FirehoseService service;private FirehoseParquetConverter parquetConverter;+ private FirehoseLambdaTransformer lambdaTransformer;private S3Service s3Service;@BeforeEach@@ -81,10 +86,11 @@ class FirehoseDataFormatConversionTest {when(regionResolver.getDefaultRegion()).thenReturn("us-east-1");when(regionResolver.getAccountId()).thenReturn("000000000000");parquetConverter = Mockito.mock(FirehoseParquetConverter.class);+ lambdaTransformer = Mockito.mock(FirehoseLambdaTransformer.class);s3Service = Mockito.mock(S3Service.class);service = new FirehoseService(storageFactory, s3Service,Mockito.mock(KinesisService.class), regionResolver, new MutableClock(), config,- parquetConverter);+ parquetConverter, lambdaTransformer);}private static SchemaConfiguration schema() {@@ -544,6 +550,57 @@ class FirehoseDataFormatConversionTest {any(byte[].class), anyString(), Mockito.anyMap(), any());}+ /**+ * A stream configured for both has to convert what the transform returned, not what+ * was put: the two were wired in separately, so nothing else would catch a delivery+ * that passed the original bytes to the converter.+ */+ @Test+ void conversionOfATransformingStreamConvertsWhatTheTransformReturned() {+ Mockito.when(lambdaTransformer.transform(any(), anyString(), any(), any()))+ .thenReturn(new FirehoseLambdaTransformer.Outcome(+ List.of("{\"ticker\": \"BBB\"}".getBytes(java.nio.charset.StandardCharsets.UTF_8)),+ 0, 0, null));+ Mockito.when(parquetConverter.deliver(any(), anyString(), any(), any()))+ .thenReturn(new FirehoseParquetConverter.Outcome(1, 0, "data/key.parquet", null));+ service.createDeliveryStream("stream", destination(s3 -> s3.setProcessingConfiguration(lambdaProcessing())));+ service.putRecord("stream", record("{\"ticker\": \"AAA\"}"));++ service.flush("stream");++ @SuppressWarnings("unchecked")+ ArgumentCaptor<List<byte[]>> converted = ArgumentCaptor.forClass(List.class);+ Mockito.verify(parquetConverter).deliver(any(), Mockito.eq("results"), converted.capture(), any());+ assertEquals(1, converted.getValue().size());+ assertEquals("{\"ticker\": \"BBB\"}",+ new String(converted.getValue().get(0), java.nio.charset.StandardCharsets.UTF_8));+ }++ @Test+ void aTransformThatKeptNothingNeverReachesTheConverter() {+ Mockito.when(lambdaTransformer.transform(any(), anyString(), any(), any()))+ .thenReturn(new FirehoseLambdaTransformer.Outcome(List.of(), 1, 0, null));+ service.createDeliveryStream("stream", destination(s3 -> s3.setProcessingConfiguration(lambdaProcessing())));+ service.putRecord("stream", record("{\"ticker\": \"AAA\"}"));++ service.flush("stream");++ Mockito.verify(parquetConverter, Mockito.never()).deliver(any(), anyString(), any(), any());+ }++ private static ProcessingConfiguration lambdaProcessing() {+ ProcessorParameter lambdaArn = new ProcessorParameter();+ lambdaArn.setParameterName("LambdaArn");+ lambdaArn.setParameterValue("arn:aws:lambda:us-east-1:000000000000:function:transform");+ Processor processor = new Processor();+ processor.setType("Lambda");+ processor.setParameters(List.of(lambdaArn));+ ProcessingConfiguration processing = new ProcessingConfiguration();+ processing.setEnabled(true);+ processing.setProcessors(List.of(processor));+ return processing;+ }+private static io.github.hectorvent.floci.services.firehose.model.Record record(String json) {io.github.hectorvent.floci.services.firehose.model.Record record =new io.github.hectorvent.floci.services.firehose.model.Record();diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformIntegrationTest.javanew file mode 100644index 0000000000..28ad6b293d--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformIntegrationTest.java@@ -0,0 +1,246 @@+package io.github.hectorvent.floci.services.firehose;++import io.github.hectorvent.floci.services.lambda.LambdaService;+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.InjectMock;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;+import org.mockito.Mockito;++import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.Base64;+import java.util.List;+import java.util.Map;+import java.util.regex.Matcher;+import java.util.regex.Pattern;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.notNullValue;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;++/**+ * End-to-end coverage of the Lambda transform on the real delivery path: what+ * the function returns is what reaches S3, and what it fails reaches the error+ * output instead. {@link FirehoseLambdaTransformerTest} covers the per-record+ * result semantics against the transformer directly; this class is what proves+ * the flush path runs it at all.+ *+ * The function itself is mocked. Running a real one needs Docker, which a+ * delivery-path test should not depend on, and the invocation payload is+ * asserted here from what the mock received.+ */+@QuarkusTest+@TestProfile(FirehoseLambdaTransformIntegrationTest.FlushEveryPairProfile.class)+class FirehoseLambdaTransformIntegrationTest {++ public static class FlushEveryPairProfile implements QuarkusTestProfile {+ @Override+ public Map<String, String> getConfigOverrides() {+ // Each test puts a batch of two, which then delivers inline.+ return Map.of("floci.services.firehose.flush-record-count", "2");+ }+ }++ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";+ private static final String TARGET_PREFIX = "Firehose_20150804.";+ private static final String FUNCTION_ARN = "arn:aws:lambda:us-east-1:000000000000:function:transform";+ private static final Pattern KEY_PATTERN = Pattern.compile("<Key>([^<]+)</Key>");++ @InjectMock+ LambdaService lambdaService;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void theFunctionsOutputIsWhatReachesTheDestination() {+ String stream = "transform-ok-stream";+ String bucket = "transform-ok-archive";+ createDeliveryStream(stream, bucket);+ respondWith((recordId, index) -> "{\"recordId\":\"" + recordId + "\",\"result\":\"Ok\",\"data\":\""+ + encode("TRANSFORMED-" + index) + "\"}");++ putRecordBatch(stream, "one", "two");++ given()+ .when()+ .get("/" + bucket + "/" + firstKey(bucket, "data/"))+ .then()+ .statusCode(200)+ .body(equalTo("TRANSFORMED-0\nTRANSFORMED-1\n"));+ }++ @Test+ void aFailedRecordGoesToTheErrorOutputWhileTheRestDelivers() {+ String stream = "transform-failed-stream";+ String bucket = "transform-failed-archive";+ createDeliveryStream(stream, bucket);+ respondWith((recordId, index) -> index == 0+ ? "{\"recordId\":\"" + recordId + "\",\"result\":\"ProcessingFailed\"}"+ : "{\"recordId\":\"" + recordId + "\",\"result\":\"Ok\",\"data\":\"" + encode("kept") + "\"}");++ putRecordBatch(stream, "one", "two");++ given()+ .when()+ .get("/" + bucket + "/" + firstKey(bucket, "data/"))+ .then()+ .statusCode(200)+ .body(equalTo("kept\n"));++ String errorLine = given()+ .when()+ .get("/" + bucket + "/" + firstKey(bucket, "errors/processing-failed/"))+ .then()+ .statusCode(200)+ .extract().asString();+ assertTrue(errorLine.contains("\"errorCode\":\"Lambda.ProcessingFailedStatus\""), errorLine);+ assertTrue(errorLine.contains("\"rawData\":\"" + encode("one") + "\""), errorLine);+ assertTrue(errorLine.contains("\"lambdaARN\":\"" + FUNCTION_ARN + "\""), errorLine);+ }++ @Test+ void aBatchTheFunctionDroppedEntirelyDeliversNothing() {+ String stream = "transform-dropped-stream";+ String bucket = "transform-dropped-archive";+ createDeliveryStream(stream, bucket);+ respondWith((recordId, index) -> "{\"recordId\":\"" + recordId + "\",\"result\":\"Dropped\"}");++ putRecordBatch(stream, "one", "two");++ String listing = given()+ .when()+ .get("/" + bucket)+ .then()+ .statusCode(200)+ .extract().asString();+ assertFalse(KEY_PATTERN.matcher(listing).find(), "expected no delivered object, got: " + listing);+ }++ @Test+ void theInvocationCarriesEveryBufferedRecord() {+ String stream = "transform-payload-stream";+ String bucket = "transform-payload-archive";+ createDeliveryStream(stream, bucket);+ List<byte[]> payloads = respondWith((recordId, index) ->+ "{\"recordId\":\"" + recordId + "\",\"result\":\"Dropped\"}");++ putRecordBatch(stream, "one", "two");++ assertEquals(1, payloads.size(), "the whole buffer should be one invocation");+ String payload = new String(payloads.get(0), StandardCharsets.UTF_8);+ assertTrue(payload.contains("\"deliveryStreamArn\":\"arn:aws:firehose:"), payload);+ assertTrue(payload.contains("\"data\":\"" + encode("one") + "\""), payload);+ assertTrue(payload.contains("\"data\":\"" + encode("two") + "\""), payload);+ }++ /**+ * Answers each invocation from the record ids it actually carries, which the caller+ * cannot name: they are minted per batch. Returns the payloads the mock received.+ */+ private List<byte[]> respondWith(RecordResponder responder) {+ List<byte[]> payloads = new ArrayList<>();+ Mockito.when(lambdaService.invokeArn(anyString(), any(), any())).thenAnswer(invocation -> {+ byte[] payload = invocation.getArgument(1);+ payloads.add(payload);+ List<String> entries = new ArrayList<>();+ Matcher recordIds = Pattern.compile("\"recordId\":\"([^\"]+)\"")+ .matcher(new String(payload, StandardCharsets.UTF_8));+ for (int index = 0; recordIds.find(); index++) {+ entries.add(responder.respond(recordIds.group(1), index));+ }+ InvokeResult result = new InvokeResult();+ result.setStatusCode(200);+ result.setPayload(("{\"records\":[" + String.join(",", entries) + "]}")+ .getBytes(StandardCharsets.UTF_8));+ return result;+ });+ return payloads;+ }++ @FunctionalInterface+ private interface RecordResponder {+ String respond(String recordId, int index);+ }++ private static String encode(String body) {+ return Base64.getEncoder().encodeToString(body.getBytes(StandardCharsets.UTF_8));+ }++ private void createDeliveryStream(String streamName, String bucket) {+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "CreateDeliveryStream")+ .body("""+ {+ "DeliveryStreamName": "%s",+ "DeliveryStreamType": "DirectPut",+ "ExtendedS3DestinationConfiguration": {+ "RoleARN": "arn:aws:iam::000000000000:role/firehose-delivery-role",+ "BucketARN": "arn:aws:s3:::%s",+ "Prefix": "data/!{timestamp:yyyy}/",+ "ErrorOutputPrefix": "errors/!{firehose:error-output-type}/",+ "ProcessingConfiguration": {+ "Enabled": true,+ "Processors": [+ {+ "Type": "Lambda",+ "Parameters": [{"ParameterName": "LambdaArn", "ParameterValue": "%s"}]+ }+ ]+ }+ }+ }+ """.formatted(streamName, bucket, FUNCTION_ARN))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DeliveryStreamARN", notNullValue());+ }++ private void putRecordBatch(String streamName, String... bodies) {+ List<String> records = new ArrayList<>();+ for (String body : bodies) {+ records.add("{\"Data\": \"" + encode(body) + "\"}");+ }+ given()+ .contentType(CONTENT_TYPE)+ .header("X-Amz-Target", TARGET_PREFIX + "PutRecordBatch")+ .body("{ \"DeliveryStreamName\": \"" + streamName + "\", \"Records\": ["+ + String.join(",", records) + "] }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("FailedPutCount", equalTo(0));+ }++ private String firstKey(String bucket, String prefix) {+ String listing = given()+ .when()+ .get("/" + bucket)+ .then()+ .statusCode(200)+ .extract().asString();+ Matcher key = KEY_PATTERN.matcher(listing);+ while (key.find()) {+ if (key.group(1).startsWith(prefix)) {+ return key.group(1);+ }+ }+ throw new AssertionError("expected an object under " + prefix + " in " + bucket + ", got: " + listing);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformerTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformerTest.javanew file mode 100644index 0000000000..ebda90bbee--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseLambdaTransformerTest.java@@ -0,0 +1,513 @@+package io.github.hectorvent.floci.services.firehose;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;+import io.github.hectorvent.floci.services.lambda.LambdaService;+import io.github.hectorvent.floci.services.lambda.model.InvocationType;+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;+import io.github.hectorvent.floci.services.s3.S3Service;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.mockito.ArgumentCaptor;++import java.nio.charset.StandardCharsets;+import java.time.Instant;+import java.util.ArrayList;+import java.util.Base64;+import java.util.List;+import java.util.Map;+import java.util.function.Function;+import java.util.stream.Collectors;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertSame;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyMap;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.verifyNoInteractions;+import static org.mockito.Mockito.when;++/**+ * The result semantics, error codes and error-object shape asserted here were+ * probed against real AWS (us-west-2, 2026-09-10), including which failures are+ * retried: a function that errors is, a response the function shaped wrongly is+ * not.+ */+class FirehoseLambdaTransformerTest {++ private static final String BUCKET = "results";+ private static final String FUNCTION_ARN = "arn:aws:lambda:us-east-1:000000000000:function:transform";+ private static final Instant DELIVERY_TIME = Instant.parse("2026-01-01T00:00:00Z");++ private LambdaService lambdaService;+ private S3Service s3Service;+ private FirehoseLambdaTransformer transformer;+ private final ObjectMapper mapper = new ObjectMapper();+ private JsonNode invocationEvent;++ @BeforeEach+ void setUp() {+ lambdaService = mock(LambdaService.class);+ s3Service = mock(S3Service.class);+ transformer = new FirehoseLambdaTransformer(lambdaService, s3Service, mapper,+ new RegionResolver("us-east-1", "000000000000"));+ }++ // ── fixtures ─────────────────────────────────────────────────────────────++ private static DeliveryStreamDescription stream(Processor... processors) {+ ProcessingConfiguration processing = new ProcessingConfiguration();+ processing.setEnabled(true);+ processing.setProcessors(List.of(processors));+ S3Destination s3 = new S3Destination();+ s3.setBucketArn("arn:aws:s3:::" + BUCKET);+ s3.setErrorOutputPrefix("errors/!{firehose:error-output-type}/");+ s3.setProcessingConfiguration(processing);+ return new DeliveryStreamDescription("stream", "arn:aws:firehose:us-west-2:000000000000:deliverystream/stream", s3);+ }++ private static Processor lambdaProcessor(String... parameters) {+ List<ProcessorParameter> list = new ArrayList<>();+ list.add(parameter("LambdaArn", FUNCTION_ARN));+ for (int i = 0; i < parameters.length; i += 2) {+ list.add(parameter(parameters[i], parameters[i + 1]));+ }+ Processor processor = new Processor();+ processor.setType("Lambda");+ processor.setParameters(list);+ return processor;+ }++ private static ProcessorParameter parameter(String name, String value) {+ ProcessorParameter parameter = new ProcessorParameter();+ parameter.setParameterName(name);+ parameter.setParameterValue(value);+ return parameter;+ }++ private static byte[] record(String body) {+ return body.getBytes(StandardCharsets.UTF_8);+ }++ private static String encode(String body) {+ return Base64.getEncoder().encodeToString(record(body));+ }++ /**+ * Answers the invocation from the record ids the transformer actually generated,+ * which is the only way a test can name them: they are minted per batch.+ */+ private void replyWith(Function<List<String>, String> responder) {+ when(lambdaService.invokeArn(anyString(), any(), any())).thenAnswer(invocation -> {+ invocationEvent = mapper.readTree((byte[]) invocation.getArgument(1));+ List<String> recordIds = new ArrayList<>();+ invocationEvent.get("records").forEach(node -> recordIds.add(node.get("recordId").asText()));+ InvokeResult result = new InvokeResult();+ result.setStatusCode(200);+ result.setPayload(responder.apply(recordIds).getBytes(StandardCharsets.UTF_8));+ return result;+ });+ }++ private static String records(String... entries) {+ return "{\"records\":[" + String.join(",", entries) + "]}";+ }++ private static String ok(String recordId, String data) {+ return "{\"recordId\":\"" + recordId + "\",\"result\":\"Ok\",\"data\":\"" + encode(data) + "\"}";+ }++ private static String outcome(String recordId, String result) {+ return "{\"recordId\":\"" + recordId + "\",\"result\":\"" + result + "\"}";+ }++ private List<String> delivered(FirehoseLambdaTransformer.Outcome outcome) {+ return outcome.records().stream()+ .map(data -> new String(data, StandardCharsets.UTF_8))+ .collect(Collectors.toList());+ }++ private List<JsonNode> errorLines() throws Exception {+ ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);+ verify(s3Service).putObject(eq(BUCKET), key.capture(), body.capture(),+ eq("application/octet-stream"), anyMap());+ assertTrue(key.getValue().startsWith("errors/processing-failed/"),+ "error output key was " + key.getValue());+ List<JsonNode> lines = new ArrayList<>();+ for (String line : new String(body.getValue(), StandardCharsets.UTF_8).split("\n")) {+ lines.add(mapper.readTree(line));+ }+ return lines;+ }++ private FirehoseLambdaTransformer.Outcome transform(DeliveryStreamDescription stream, byte[]... records) {+ return transformer.transform(stream, BUCKET, List.of(records), DELIVERY_TIME);+ }++ // ── pass-through ─────────────────────────────────────────────────────────++ @Test+ void aStreamWithoutAProcessingConfigurationIsNotTransformed() {+ S3Destination s3 = new S3Destination();+ s3.setBucketArn("arn:aws:s3:::" + BUCKET);+ DeliveryStreamDescription stream =+ new DeliveryStreamDescription("stream", "arn:aws:firehose:::stream", s3);+ List<byte[]> records = List.of(record("a"));++ FirehoseLambdaTransformer.Outcome outcome =+ transformer.transform(stream, BUCKET, records, DELIVERY_TIME);++ assertSame(records, outcome.records());+ verifyNoInteractions(lambdaService);+ }++ /**+ * Probed 2026-09-11: an omitted Enabled is stored as false, so it leaves the stream+ * untransformed. The conversion block treats an omitted Enabled the other way, which+ * is why this is asserted rather than inferred from it.+ */+ @Test+ void aProcessingConfigurationWithoutAnExplicitEnabledIsNotTransformed() {+ DeliveryStreamDescription stream = stream(lambdaProcessor());+ stream.s3Destination().getProcessingConfiguration().setEnabled(null);++ FirehoseLambdaTransformer.Outcome outcome = transform(stream, record("a"));++ assertEquals(List.of("a"), delivered(outcome));+ verifyNoInteractions(lambdaService);+ }++ @Test+ void aDisabledProcessingConfigurationIsNotTransformed() {+ DeliveryStreamDescription stream = stream(lambdaProcessor());+ stream.s3Destination().getProcessingConfiguration().setEnabled(false);++ transform(stream, record("a"));++ verifyNoInteractions(lambdaService);+ }++ @Test+ void aProcessorTypeFlociDoesNotApplyLeavesTheRecordsAlone() {+ Processor appendDelimiter = new Processor();+ appendDelimiter.setType("AppendDelimiterToRecord");++ FirehoseLambdaTransformer.Outcome outcome = transform(stream(appendDelimiter), record("a"));++ assertEquals(List.of("a"), delivered(outcome));+ verifyNoInteractions(lambdaService);+ }++ // ── invocation payload ───────────────────────────────────────────────────++ @Test+ void theInvocationCarriesTheProbedEventShape() {+ replyWith(ids -> records(ok(ids.get(0), "transformed")));++ transform(stream(lambdaProcessor()), record("original"));++ assertEquals(List.of("invocationId", "deliveryStreamArn", "region", "records"),+ fieldNames(invocationEvent));+ assertEquals("arn:aws:firehose:us-west-2:000000000000:deliverystream/stream",+ invocationEvent.get("deliveryStreamArn").asText());+ assertEquals("us-west-2", invocationEvent.get("region").asText());+ JsonNode record = invocationEvent.get("records").get(0);+ assertEquals(List.of("recordId", "approximateArrivalTimestamp", "data"), fieldNames(record));+ assertEquals(DELIVERY_TIME.toEpochMilli(), record.get("approximateArrivalTimestamp").asLong());+ assertEquals(encode("original"), record.get("data").asText());+ verify(lambdaService).invokeArn(eq(FUNCTION_ARN), any(), eq(InvocationType.RequestResponse));+ }++ private static List<String> fieldNames(JsonNode node) {+ List<String> names = new ArrayList<>();+ node.fieldNames().forEachRemaining(names::add);+ return names;+ }++ // ── per-record results ───────────────────────────────────────────────────++ @Test+ void okReplacesTheRecordWithTheReturnedData() {+ replyWith(ids -> records(ok(ids.get(0), "one"), ok(ids.get(1), "two")));++ FirehoseLambdaTransformer.Outcome outcome =+ transform(stream(lambdaProcessor()), record("a"), record("b"));++ assertEquals(List.of("one", "two"), delivered(outcome));+ assertEquals(0, outcome.failedRecords());+ assertNull(outcome.errorKey());+ verify(s3Service, never()).putObject(anyString(), anyString(), any(), anyString(), anyMap());+ }++ @Test+ void droppedLeavesNoRecordAndNoErrorObject() {+ replyWith(ids -> records(outcome(ids.get(0), "Dropped"), ok(ids.get(1), "kept")));++ FirehoseLambdaTransformer.Outcome outcome =+ transform(stream(lambdaProcessor()), record("a"), record("b"));++ assertEquals(List.of("kept"), delivered(outcome));+ assertEquals(1, outcome.droppedRecords());+ assertEquals(0, outcome.failedRecords());+ assertNull(outcome.errorKey());+ }++ @Test+ void processingFailedRoutesOnlyThatRecordToTheErrorOutput() throws Exception {+ replyWith(ids -> records(outcome(ids.get(0), "ProcessingFailed"), ok(ids.get(1), "kept")));++ FirehoseLambdaTransformer.Outcome outcome =+ transform(stream(lambdaProcessor()), record("a"), record("b"));++ assertEquals(List.of("kept"), delivered(outcome));+ assertEquals(1, outcome.failedRecords());+ JsonNode line = errorLines().get(0);+ assertEquals("Lambda.ProcessingFailedStatus", line.get("errorCode").asText());+ assertEquals("ProcessingFailed status set for record", line.get("errorMessage").asText());+ assertEquals(encode("a"), line.get("rawData").asText());+ }++ @Test+ void aRecordTheFunctionDidNotReturnFails() throws Exception {+ replyWith(ids -> records(ok(ids.get(0), "one")));++ transform(stream(lambdaProcessor()), record("a"), record("b"));++ JsonNode line = errorLines().get(0);+ assertEquals("Lambda.MissingRecordId", line.get("errorCode").asText());+ assertEquals(encode("b"), line.get("rawData").asText());+ }++ @Test+ void aRecordIdReturnedTwiceFails() throws Exception {+ replyWith(ids -> records(ok(ids.get(0), "one"), ok(ids.get(0), "again"), ok(ids.get(1), "two")));++ FirehoseLambdaTransformer.Outcome outcome =+ transform(stream(lambdaProcessor()), record("a"), record("b"));++ assertEquals(List.of("two"), delivered(outcome));+ assertEquals("Lambda.DuplicatedRecordId", errorLines().get(0).get("errorCode").asText());+ }++ @Test+ void aRecordIdTheBatchNeverCarriedIsIgnored() {+ replyWith(ids -> records(ok(ids.get(0), "one"), ok("not-in-this-batch", "stray")));++ FirehoseLambdaTransformer.Outcome outcome = transform(stream(lambdaProcessor()), record("a"));++ assertEquals(List.of("one"), delivered(outcome));+ assertEquals(0, outcome.failedRecords());+ }++ @Test+ void okWithoutDataFails() throws Exception {+ replyWith(ids -> records(outcome(ids.get(0), "Ok")));++ transform(stream(lambdaProcessor()), record("a"));++ JsonNode line = errorLines().get(0);+ assertEquals("Lambda.InvalidReturnFormat", line.get("errorCode").asText());+ assertTrue(line.get("errorMessage").asText().startsWith("The data field cannot be null"));+ }++ /**+ * The message here is Floci's own: AWS was not probed for a data field that is not+ * base64 at all, only for one that is absent.+ */+ @Test+ void dataThatIsNotBase64Fails() throws Exception {+ replyWith(ids -> records(+ "{\"recordId\":\"" + ids.get(0) + "\",\"result\":\"Ok\",\"data\":\"not base64 %%%\"}",+ ok(ids.get(1), "kept")));++ FirehoseLambdaTransformer.Outcome outcome =+ transform(stream(lambdaProcessor()), record("a"), record("b"));++ assertEquals(List.of("kept"), delivered(outcome));+ assertEquals(1, outcome.failedRecords());+ JsonNode line = errorLines().get(0);+ assertEquals("Lambda.InvalidReturnFormat", line.get("errorCode").asText());+ assertTrue(line.get("errorMessage").asText().startsWith("The data field must be base64 encoded"));+ assertEquals(encode("a"), line.get("rawData").asText());+ }++ @Test+ void anUnknownResultValueFails() throws Exception {+ replyWith(ids -> records(outcome(ids.get(0), "Maybe")));++ transform(stream(lambdaProcessor()), record("a"));++ assertEquals("Lambda.InvalidReturnFormat", errorLines().get(0).get("errorCode").asText());+ }++ // ── failures of the invocation itself ────────────────────────────────────++ @Test+ void aFunctionErrorIsRetriedAndThenFailsTheWholeBatch() throws Exception {+ InvokeResult errored = new InvokeResult();+ errored.setStatusCode(200);+ errored.setFunctionError("Unhandled");+ when(lambdaService.invokeArn(anyString(), any(), any())).thenReturn(errored);++ FirehoseLambdaTransformer.Outcome outcome =+ transform(stream(lambdaProcessor("NumberOfRetries", "2")), record("a"), record("b"));++ assertTrue(outcome.records().isEmpty());+ assertEquals(2, outcome.failedRecords());+ verify(lambdaService, times(3)).invokeArn(anyString(), any(), any());+ JsonNode line = errorLines().get(0);+ assertEquals("Lambda.FunctionError", line.get("errorCode").asText());+ assertEquals(3, line.get("attemptsMade").asInt());+ }++ @Test+ void anUninvokableFunctionIsRetriedTheSameWay() throws Exception {+ when(lambdaService.invokeArn(anyString(), any(), any()))+ .thenThrow(new AwsException("ResourceNotFoundException", "Function not found", 404));++ transform(stream(lambdaProcessor("NumberOfRetries", "1")), record("a"));++ verify(lambdaService, times(2)).invokeArn(anyString(), any(), any());+ assertEquals("Lambda.FunctionError", errorLines().get(0).get("errorCode").asText());+ }++ @Test+ void aResponseCarryingNoRecordsIsNotRetried() throws Exception {+ replyWith(ids -> "{}");++ transform(stream(lambdaProcessor("NumberOfRetries", "5")), record("a"));++ verify(lambdaService, times(1)).invokeArn(anyString(), any(), any());+ JsonNode line = errorLines().get(0);+ assertEquals("Lambda.MissingRecordId", line.get("errorCode").asText());+ assertEquals(1, line.get("attemptsMade").asInt());+ }++ @Test+ void anUnparseableResponseFailsEveryRecordWithoutRetrying() throws Exception {+ replyWith(ids -> "not json at all");++ transform(stream(lambdaProcessor()), record("a"));++ verify(lambdaService, times(1)).invokeArn(anyString(), any(), any());+ assertEquals("Lambda.MissingRecordId", errorLines().get(0).get("errorCode").asText());+ }++ @Test+ void anAbsentNumberOfRetriesFallsBackToTheThreeAwsDefaults() {+ InvokeResult errored = new InvokeResult();+ errored.setFunctionError("Unhandled");+ when(lambdaService.invokeArn(anyString(), any(), any())).thenReturn(errored);++ transform(stream(lambdaProcessor()), record("a"));++ verify(lambdaService, times(4)).invokeArn(anyString(), any(), any());+ }++ /**+ * AWS stores a NumberOfRetries it never range-checks, so the largest one a caller can+ * write has to stay a bounded number of invocations rather than overflowing the+ * attempt count or occupying the flusher thread indefinitely.+ */+ @Test+ void aRetryCountLargerThanFlociHonoursIsCapped() throws Exception {+ InvokeResult errored = new InvokeResult();+ errored.setFunctionError("Unhandled");+ when(lambdaService.invokeArn(anyString(), any(), any())).thenReturn(errored);++ transform(stream(lambdaProcessor("NumberOfRetries", String.valueOf(Integer.MAX_VALUE))), record("a"));++ verify(lambdaService, times(101)).invokeArn(anyString(), any(), any());+ assertEquals(101, errorLines().get(0).get("attemptsMade").asInt());+ }++ /**+ * A NumberOfRetries too wide even for a long is still a number, and treating it as+ * unparseable would hand it fewer attempts than a smaller one gets.+ */+ @Test+ void aRetryCountTooWideForALongIsCappedRatherThanDefaulted() {+ InvokeResult errored = new InvokeResult();+ errored.setFunctionError("Unhandled");+ when(lambdaService.invokeArn(anyString(), any(), any())).thenReturn(errored);++ transform(stream(lambdaProcessor("NumberOfRetries", "999999999999999999999")), record("a"));++ verify(lambdaService, times(101)).invokeArn(anyString(), any(), any());+ }++ @Test+ void aNegativeRetryCountInvokesOnce() {+ replyWith(ids -> records(ok(ids.get(0), "one")));++ transform(stream(lambdaProcessor("NumberOfRetries", "-5")), record("a"));++ verify(lambdaService, times(1)).invokeArn(anyString(), any(), any());+ }++ // ── error object ─────────────────────────────────────────────────────────++ @Test+ void theErrorObjectFollowsTheProbedMemberOrder() throws Exception {+ replyWith(ids -> records(outcome(ids.get(0), "ProcessingFailed")));++ transform(stream(lambdaProcessor()), record("a"));++ JsonNode line = errorLines().get(0);+ assertEquals(List.of("rawData", "errorCode", "errorMessage", "attemptsMade",+ "arrivalTimestamp", "attemptEndingTimestamp", "lambdaARN"),+ fieldNames(line));+ assertEquals(FUNCTION_ARN, line.get("lambdaARN").asText());+ assertEquals(DELIVERY_TIME.toEpochMilli(), line.get("arrivalTimestamp").asLong());+ assertEquals(DELIVERY_TIME.toEpochMilli(), line.get("attemptEndingTimestamp").asLong());+ }++ @Test+ void everyFailedRecordGetsItsOwnLineInOneErrorObject() throws Exception {+ replyWith(ids -> records(outcome(ids.get(0), "ProcessingFailed"), outcome(ids.get(1), "ProcessingFailed")));++ FirehoseLambdaTransformer.Outcome outcome =+ transform(stream(lambdaProcessor()), record("a"), record("b"));++ assertEquals(2, outcome.failedRecords());+ List<JsonNode> lines = errorLines();+ assertEquals(2, lines.size());+ assertEquals(encode("a"), lines.get(0).get("rawData").asText());+ assertEquals(encode("b"), lines.get(1).get("rawData").asText());+ }++ @Test+ void theErrorOutputKeyResolvesTheErrorOutputTypeExpression() {+ replyWith(ids -> records(outcome(ids.get(0), "ProcessingFailed")));++ FirehoseLambdaTransformer.Outcome outcome = transform(stream(lambdaProcessor()), record("a"));++ assertTrue(outcome.errorKey().startsWith("errors/processing-failed/stream-"),+ "error output key was " + outcome.errorKey());+ }++ @Test+ void theBatchIsWrittenToTheDestinationBucket() {+ replyWith(ids -> records(outcome(ids.get(0), "ProcessingFailed")));++ transform(stream(lambdaProcessor()), record("a"));++ verify(s3Service).putObject(eq(BUCKET), anyString(), any(), eq("application/octet-stream"),+ eq(Map.of()));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseProcessingConfigurationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseProcessingConfigurationTest.javaindex 09f03d8c95..4c51294a87 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseProcessingConfigurationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseProcessingConfigurationTest.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessingConfiguration;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.Processor;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.ProcessorParameter;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.BufferingHints;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;import io.github.hectorvent.floci.services.kinesis.KinesisService;import io.github.hectorvent.floci.services.s3.S3Service;@@ -26,6 +27,7 @@ import java.util.function.Consumer;import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;@@ -70,7 +72,8 @@ class FirehoseProcessingConfigurationTest {when(regionResolver.getAccountId()).thenReturn("000000000000");service = new FirehoseService(storageFactory, mock(S3Service.class),Mockito.mock(KinesisService.class), regionResolver, new MutableClock(), config,- Mockito.mock(FirehoseParquetConverter.class));+ Mockito.mock(FirehoseParquetConverter.class),+ Mockito.mock(FirehoseLambdaTransformer.class));}private static ProcessorParameter parameter(String name, String value) {@@ -94,6 +97,20 @@ class FirehoseProcessingConfigurationTest {return processing("Lambda", parameters);}+ private static Processor lambdaProcessor(ProcessorParameter... parameters) {+ Processor processor = new Processor();+ processor.setType("Lambda");+ processor.setParameters(new ArrayList<>(List.of(parameters)));+ return processor;+ }++ private static BufferingHints hints(int sizeInMBs, int intervalInSeconds) {+ BufferingHints hints = new BufferingHints();+ hints.setSizeInMBs(sizeInMBs);+ hints.setIntervalInSeconds(intervalInSeconds);+ return hints;+ }+private static S3Destination destination(Consumer<S3Destination> customizer) {S3Destination s3 = new S3Destination();s3.setRoleArn(ROLE_ARN);@@ -329,7 +346,69 @@ class FirehoseProcessingConfigurationTest {service.updateDestination("stream", currentVersion("stream"), DESTINATION_ID,destination(s3 -> s3.setPrefix("changed/")));- assertEquals(List.of("LambdaArn", "NumberOfRetries", "RoleArn"), storedParameterNames("stream"));+ assertEquals(List.of("LambdaArn", "NumberOfRetries", "RoleArn", "BufferSizeInMBs",+ "BufferIntervalInSeconds"), storedParameterNames("stream"));+ }++ /**+ * Probed 2026-09-11: DescribeDeliveryStream echoes an omitted Enabled back as false+ * here, where the conversion block echoes it as true, so the two cannot share a rule.+ */+ @Test+ void anOmittedEnabledIsStoredAsDisabled() {+ ProcessingConfiguration processing = new ProcessingConfiguration();+ processing.setProcessors(List.of(lambdaProcessor(parameter("LambdaArn", LAMBDA_ARN))));+ service.createDeliveryStream("stream",+ destination(s3 -> s3.setProcessingConfiguration(processing)));++ S3Destination stored = service.describeDeliveryStream("stream").s3Destination();+ assertEquals(false, stored.getProcessingConfiguration().getEnabled());+ assertFalse(stored.isProcessingEnabled());+ }++ /**+ * The two buffer parameters are the Lambda processor's own defaults, not the+ * destination's BufferingHints: probed, a destination buffering 5 MiB over 300s still+ * echoes 1 and 60 on the processor.+ */+ @Test+ void aProcessorWithoutBufferParametersGainsTheLambdaDefaults() {+ service.createDeliveryStream("stream", destination(s3 -> {+ s3.setBufferingHints(hints(5, 300));+ s3.setProcessingConfiguration(lambdaProcessing(parameter("LambdaArn", LAMBDA_ARN)));+ }));++ List<ProcessorParameter> stored = service.describeDeliveryStream("stream").s3Destination()+ .getProcessingConfiguration().getProcessors().get(0).getParameters();+ assertEquals("1", parameterValue(stored, "BufferSizeInMBs"));+ assertEquals("60", parameterValue(stored, "BufferIntervalInSeconds"));+ }++ /**+ * State persisted before Enabled was defaulted never passes canonicalizeProcessors+ * again, so the describe path heals it rather than reporting a stream without the+ * member AWS always returns.+ */+ @Test+ void applyDefaultsHealsAProcessingBlockStoredWithoutEnabled() {+ ProcessingConfiguration processing = new ProcessingConfiguration();+ processing.setProcessors(List.of(lambdaProcessor(parameter("LambdaArn", LAMBDA_ARN))));+ S3Destination stored = new S3Destination();+ stored.setProcessingConfiguration(processing);++ stored.applyDefaults();++ assertEquals(false, processing.getEnabled());+ assertFalse(stored.isProcessingEnabled());+ }++ private static String parameterValue(List<ProcessorParameter> parameters, String name) {+ for (ProcessorParameter parameter : parameters) {+ if (name.equals(parameter.getParameterName())) {+ return parameter.getParameterValue();+ }+ }+ return null;}// Validation runs against the update's own content, since the update replaces thediff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.javaindex 9987fa65dc..97f895c3d5 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java@@ -91,7 +91,8 @@ class FirehoseServiceTest {// sees what a GetRecords consumer sees, and a stubbed one could not show that.kinesisService = new KinesisService(storageFactory, regionResolver);return new FirehoseService(storageFactory, s3Service, kinesisService,- regionResolver, clock, config, mock(FirehoseParquetConverter.class));+ regionResolver, clock, config, mock(FirehoseParquetConverter.class),+ mock(FirehoseLambdaTransformer.class));}private static S3Destination destination(String bucketArn, String compressionFormat) {