Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(stepfunctions): emulate distributed Map ResultWriter (S3 export + manifest) (#1823)

Alex Banna59cb643d6e28
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
index 17153d062a..b94af9e128 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutor.java
@@ -1463,14 +1463,24 @@ public class AslExecutor {
private StateResult executeMapState(String name, JsonNode stateDef, JsonNode input, StateMachine sm,
boolean jsonata, String topLevelQueryLanguage, JsonNode context,
ObjectNode variables) throws Exception {
- if (stateDef.has("ItemReader")) {
+ // ItemReader, ItemBatcher and ResultWriter are distributed-map-only fields; AWS rejects them on
+ // an INLINE map. Guard on all three (not just ItemReader) so a ResultWriter on an inline map is
+ // rejected rather than silently exporting to S3.
+ if (stateDef.has("ItemReader") || stateDef.has("ItemBatcher") || stateDef.has("ResultWriter")) {
String mode = stateDef.path("ItemProcessor").path("ProcessorConfig").path("Mode").asText("INLINE");
if (!"DISTRIBUTED".equals(mode)) {
throw new FailStateException("States.Runtime",
"The ItemReader, ItemBatcher and ResultWriter fields are not supported for INLINE maps");
}
}
- ResolvedMapItems resolvedItems = resolveMapItems(stateDef, input, jsonata, context, variables);
+ boolean hasResultWriter = stateDef.has("ResultWriter");
+ // ResultWriter Parameters and its distributed Map item selection see the effective input
+ // after InputPath. Keep the pre-existing item-selection behavior unchanged for Maps that do
+ // not use ResultWriter so this feature does not absorb a separate compatibility correction.
+ JsonNode mapInput = jsonata ? input : applyInputPath(stateDef, input);
+ JsonNode itemSelectionInput = hasResultWriter ? mapInput : input;
+ ResolvedMapItems resolvedItems = resolveMapItems(
+ stateDef, itemSelectionInput, jsonata, context, variables);
JsonNode items = resolvedItems.items();
if (!items.isArray()) {
@@ -1486,10 +1496,11 @@ public class AslExecutor {
JsonNode itemTransform = stateDef.has("ItemSelector") ? stateDef.get("ItemSelector")
: stateDef.has("Parameters") ? stateDef.get("Parameters") : null;
- // Resolve InputPath before iterating so $. in ItemSelector sees the Map state's effective input
- JsonNode mapInput = applyInputPath(stateDef, input);
-
ArrayNode results = objectMapper.createArrayNode();
+ // When a ResultWriter exports results, AWS records each child execution's input and timing
+ // (Transformation NONE); capture them alongside the outputs so the export is faithful.
+ ArrayNode childInputs = hasResultWriter ? objectMapper.createArrayNode() : null;
+ List<long[]> childTimings = hasResultWriter ? new ArrayList<>() : null;
int index = 0;
for (JsonNode item : items) {
ObjectNode iterContext = ((ObjectNode) context).deepCopy();
@@ -1512,25 +1523,279 @@ public class AslExecutor {
}
// Each iteration gets an isolated copy of the current variables; assignments inside an
// iteration are scoped to that iteration and do not leak back to the parent scope.
- results.add(executeBranch(startAt, iteratorStates, iterInput, sm, topLevelQueryLanguage, iterContext,
- variables.deepCopy()));
+ long startMs = hasResultWriter ? System.currentTimeMillis() : 0L;
+ JsonNode branchOutput =
+ executeBranch(startAt, iteratorStates, iterInput, sm, topLevelQueryLanguage, iterContext,
+ variables.deepCopy());
+ if (hasResultWriter) {
+ childInputs.add(iterInput);
+ childTimings.add(new long[]{startMs, System.currentTimeMillis()});
+ }
+ results.add(branchOutput);
index++;
}
+ // A distributed Map's ResultWriter formats the per-child results and, when an S3 Resource is
+ // given, exports them and yields a reference to the manifest instead of the inline array.
+ JsonNode mapResult = results;
+ if (hasResultWriter) {
+ mapResult = applyResultWriter(name, stateDef, mapInput, results, childInputs, childTimings,
+ sm, context, jsonata, variables);
+ }
+
if (jsonata) {
- JsonNode output = applyJsonataOutput(stateDef, input, results, context, variables);
+ JsonNode output = applyJsonataOutput(stateDef, input, mapResult, context, variables);
return new StateResult(output, stateDef.path("Next").asText(null));
}
// ResultSelector transforms the raw iteration results before ResultPath merges them in.
JsonNode selected = stateDef.has("ResultSelector")
- ? resolveParameters(stateDef.get("ResultSelector"), results, context)
- : results;
+ ? resolveParameters(stateDef.get("ResultSelector"), mapResult, context)
+ : mapResult;
JsonNode output = mergeResult(stateDef, input, selected);
output = applyOutputPath(stateDef, input, output);
return new StateResult(output, stateDef.path("Next").asText(null));
}
+ /**
+ * Emulates a Distributed Map state's {@code ResultWriter}
+ * (<a href="https://docs.aws.amazon.com/step-functions/latest/dg/input-output-resultwriter.html">AWS docs</a>).
+ * The child results are formatted per {@code WriterConfig.Transformation} ({@code NONE} /
+ * {@code COMPACT} / {@code FLATTEN}); then:
+ * <ul>
+ * <li>if a {@code Resource} + {@code Parameters}/{@code Arguments} (an S3 bucket/prefix) are
+ * given, the formatted results are exported to S3 as {@code SUCCEEDED_n.json} plus a
+ * {@code manifest.json}, and the Map state returns
+ * {@code {MapRunArn, ResultWriterDetails:{Bucket, Key}}};</li>
+ * <li>if only a {@code WriterConfig} is given (no S3 {@code Resource}), the formatted results
+ * are returned directly to the next state (preview) with no S3 write.</li>
+ * </ul>
+ *
+ * <p>By construction every child branch here has already succeeded (a failed branch throws and
+ * fails the Map before this point, since inline Maps here do not implement tolerated-failure),
+ * so {@code ResultFiles.FAILED} / {@code PENDING} are empty and all results go to a single
+ * {@code SUCCEEDED_0.json}.
+ */
+ // Package-private for unit testing of the ResultWriter export/format behaviour.
+ JsonNode applyResultWriter(String mapStateName, JsonNode stateDef, JsonNode input,
+ ArrayNode results, ArrayNode childInputs, List<long[]> childTimings,
+ StateMachine sm, JsonNode context, boolean jsonata) throws Exception {
+ return applyResultWriter(mapStateName, stateDef, input, results, childInputs, childTimings,
+ sm, context, jsonata, objectMapper.createObjectNode());
+ }
+
+ private JsonNode applyResultWriter(String mapStateName, JsonNode stateDef, JsonNode input,
+ ArrayNode results, ArrayNode childInputs, List<long[]> childTimings,
+ StateMachine sm, JsonNode context, boolean jsonata,
+ ObjectNode variables) throws Exception {
+ JsonNode writer = stateDef.get("ResultWriter");
+ JsonNode writerConfig = writer.path("WriterConfig");
+ boolean export = writer.hasNonNull("Resource");
+ // When exporting without an explicit WriterConfig, AWS defaults the transformation to NONE
+ // (child results plus execution metadata); COMPACT is the default only when no ResultWriter
+ // is present at all, which is handled by returning the inline array elsewhere.
+ String transformation = writerConfig.path("Transformation").asText("NONE");
+ String outputType = writerConfig.path("OutputType").asText("JSON");
+
+ String region = extractRegionFromArn(sm.getStateMachineArn());
+ String account = AwsArnUtils.accountOrDefault(sm.getStateMachineArn(), null);
+ String smName = context.path("StateMachine").path("Name").asText(sm.getName());
+ String mapRunLabel = stateDef.path("Label").asText(null);
+ if (mapRunLabel == null || mapRunLabel.isBlank()) {
+ mapRunLabel = UUID.randomUUID().toString();
+ }
+
+ JsonNode formatted = formatMapResults(transformation, results, childInputs, childTimings,
+ region, account, smName, mapRunLabel);
+
+ if (!export) {
+ // WriterConfig only: return the formatted results to the next state (no S3 write).
+ return formatted;
+ }
+
+ try {
+ // Resolve the destination bucket/prefix: JSONata states carry them under Arguments,
+ // JSONPath states under Parameters. Reference paths see the Map's effective input after
+ // InputPath, which is supplied by executeMapState.
+ JsonNode loc;
+ if (jsonata && writer.has("Arguments")) {
+ JsonNode arguments = writer.get("Arguments");
+ loc = jsonataEvaluator.resolveTemplate(
+ arguments, buildStatesVar(input, null, context), variables);
+ if (arguments.isObject()) {
+ if (arguments.has("Bucket") && !loc.has("Bucket")) {
+ throw new FailStateException("States.QueryEvaluationError",
+ "ResultWriter Bucket must resolve to a string");
+ }
+ if (arguments.has("Prefix") && !loc.has("Prefix")) {
+ throw new FailStateException("States.QueryEvaluationError",
+ "ResultWriter Prefix must resolve to a string");
+ }
+ }
+ } else if (writer.has("Parameters")) {
+ loc = resolveParameters(writer.get("Parameters"), input, context);
+ } else {
+ loc = objectMapper.createObjectNode();
+ }
+ if (!loc.isObject()) {
+ throw new FailStateException(
+ jsonata ? "States.QueryEvaluationError" : "States.ResultWriterFailed",
+ "ResultWriter " + (jsonata ? "Arguments" : "Parameters")
+ + " must resolve to an object");
+ }
+ JsonNode bucketNode = loc.get("Bucket");
+ if (bucketNode == null) {
+ throw new FailStateException("States.ResultWriterFailed",
+ "ResultWriter destination bucket is required");
+ }
+ if (!bucketNode.isTextual()) {
+ throw new FailStateException(
+ jsonata ? "States.QueryEvaluationError" : "States.ResultWriterFailed",
+ "ResultWriter Bucket must resolve to a string");
+ }
+ String bucket = bucketNode.asText();
+ if (bucket.isBlank()) {
+ throw new FailStateException("States.ResultWriterFailed",
+ "ResultWriter destination bucket is required");
+ }
+ JsonNode prefixNode = loc.get("Prefix");
+ if (prefixNode != null && !prefixNode.isTextual()) {
+ throw new FailStateException(
+ jsonata ? "States.QueryEvaluationError" : "States.ResultWriterFailed",
+ "ResultWriter Prefix must resolve to a string");
+ }
+ String prefix = prefixNode == null ? "" : prefixNode.asText();
+
+ // S3 buckets are owned by Floci's single synthetic account. AWS additionally requires
+ // ResultWriter destinations to be in the state machine's Region.
+ String bucketRegion = normalizeS3Region(s3Service.getBucketRegion(bucket));
+ if (!bucketRegion.equalsIgnoreCase(region)) {
+ throw new FailStateException("States.ResultWriterFailed",
+ "ResultWriter destination bucket must be in the same AWS Region "
+ + "as the state machine");
+ }
+
+ // AWS includes the Map label (or an automatically generated label) before the run id.
+ // The run id alone keys the exported result set under the user-supplied S3 prefix.
+ String mapRunId = UUID.randomUUID().toString();
+ String mapRunArn = "arn:aws:states:" + region + ":" + account + ":mapRun:"
+ + smName + "/" + mapRunLabel + ":" + mapRunId;
+ String base = prefix.isEmpty()
+ ? mapRunId + "/"
+ : prefix + (prefix.endsWith("/") ? "" : "/") + mapRunId + "/";
+
+ String succeededKey = base + "SUCCEEDED_0.json";
+ String manifestKey = base + "manifest.json";
+
+ byte[] succeededBytes = serializeResultFile(formatted, outputType);
+ s3Service.putObject(bucket, succeededKey, succeededBytes, "application/json", new HashMap<>());
+
+ ObjectNode manifest = objectMapper.createObjectNode();
+ manifest.put("DestinationBucket", bucket);
+ manifest.put("MapRunArn", mapRunArn);
+ ObjectNode resultFiles = manifest.putObject("ResultFiles");
+ resultFiles.putArray("FAILED");
+ resultFiles.putArray("PENDING");
+ ObjectNode succeededEntry = resultFiles.putArray("SUCCEEDED").addObject();
+ succeededEntry.put("Key", succeededKey);
+ succeededEntry.put("Size", succeededBytes.length);
+ s3Service.putObject(bucket, manifestKey, objectMapper.writeValueAsBytes(manifest),
+ "application/json", new HashMap<>());
+
+ ObjectNode mapResult = objectMapper.createObjectNode();
+ mapResult.put("MapRunArn", mapRunArn);
+ ObjectNode details = mapResult.putObject("ResultWriterDetails");
+ details.put("Bucket", bucket);
+ details.put("Key", manifestKey);
+ return mapResult;
+ } catch (FailStateException e) {
+ throw e;
+ } catch (Exception e) {
+ String detail = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName();
+ throw new FailStateException("States.ResultWriterFailed",
+ "Unable to export Map Run results: " + detail);
+ }
+ }
+
+ /**
+ * Formats a Distributed Map's child results per {@code WriterConfig.Transformation}:
+ * <ul>
+ * <li>{@code NONE} — an array of per-child execution records (output plus execution metadata),
+ * as AWS writes when exporting without a WriterConfig;</li>
+ * <li>{@code COMPACT} — the child outputs in their original array structure;</li>
+ * <li>{@code FLATTEN} — the child outputs with any array outputs flattened into one array.</li>
+ * </ul>
+ */
+ private ArrayNode formatMapResults(String transformation, ArrayNode results, ArrayNode childInputs,
+ List<long[]> childTimings, String region, String account,
+ String smName, String mapRunLabel) {
+ ArrayNode out = objectMapper.createArrayNode();
+ if ("FLATTEN".equalsIgnoreCase(transformation)) {
+ for (JsonNode r : results) {
+ if (r.isArray()) {
+ r.forEach(out::add);
+ } else {
+ out.add(r);
+ }
+ }
+ return out;
+ }
+ if ("COMPACT".equalsIgnoreCase(transformation)) {
+ out.addAll(results);
+ return out;
+ }
+ // NONE: emit an execution record per child, mirroring the AWS export format. The child
+ // executions run under a derived state machine "<parentName>/<mapRunLabel>".
+ String childSmArn = "arn:aws:states:" + region + ":" + account + ":stateMachine:"
+ + smName + "/" + mapRunLabel;
+ for (int i = 0; i < results.size(); i++) {
+ String childId = java.util.UUID.randomUUID().toString();
+ long start = childTimings != null && i < childTimings.size() ? childTimings.get(i)[0] : 0L;
+ long stop = childTimings != null && i < childTimings.size() ? childTimings.get(i)[1] : 0L;
+ ObjectNode rec = out.addObject();
+ rec.put("ExecutionArn", "arn:aws:states:" + region + ":" + account + ":execution:"
+ + smName + "/" + mapRunLabel + ":" + childId);
+ rec.put("Input", stringifyResult(childInputs != null && i < childInputs.size()
+ ? childInputs.get(i) : NullNode.getInstance()));
+ rec.putObject("InputDetails").put("Included", true);
+ rec.put("Name", childId);
+ rec.put("Output", stringifyResult(results.get(i)));
+ rec.putObject("OutputDetails").put("Included", true);
+ rec.put("RedriveCount", 0);
+ rec.put("RedriveStatus", "NOT_REDRIVABLE");
+ rec.put("RedriveStatusReason", "Execution is SUCCEEDED and cannot be redriven");
+ rec.put("StartDate", java.time.Instant.ofEpochMilli(start).toString());
+ rec.put("StateMachineArn", childSmArn);
+ rec.put("Status", "SUCCEEDED");
+ rec.put("StopDate", java.time.Instant.ofEpochMilli(stop).toString());
+ }
+ return out;
+ }
+
+ /**
+ * AWS records a child execution's Input/Output as a JSON-encoded string. Using {@code toString()}
+ * (not {@code asText()}) keeps scalar strings quoted — the string {@code hello} is exported as
+ * {@code "hello"} — so the field is always valid JSON regardless of the value's type.
+ */
+ private String stringifyResult(JsonNode node) {
+ if (node == null || node.isNull() || node.isMissingNode()) {
+ return "null";
+ }
+ return node.toString();
+ }
+
+ /** Serializes a formatted result array as a JSON array or as JSON Lines per {@code OutputType}. */
+ private byte[] serializeResultFile(JsonNode formatted, String outputType) throws Exception {
+ if ("JSONL".equalsIgnoreCase(outputType) && formatted.isArray()) {
+ StringBuilder sb = new StringBuilder();
+ for (JsonNode element : formatted) {
+ sb.append(objectMapper.writeValueAsString(element)).append('\n');
+ }
+ return sb.toString().getBytes(StandardCharsets.UTF_8);
+ }
+ return objectMapper.writeValueAsBytes(formatted);
+ }
+
private ResolvedMapItems resolveMapItems(JsonNode stateDef, JsonNode input,
boolean jsonata, JsonNode context, ObjectNode variables) throws Exception {
if (jsonata && stateDef.has("Items")) {
@@ -2652,6 +2917,10 @@ public class AslExecutor {
return AwsArnUtils.regionOrDefault(arn, "us-east-1");
}
+ private static String normalizeS3Region(String region) {
+ return region == null || region.isBlank() ? "us-east-1" : region;
+ }
+
record StateResult(JsonNode output, String nextState) {}
static class FailStateException extends RuntimeException {
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
index b756990b24..9cf42c7503 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java
@@ -54,6 +54,9 @@ public class StepFunctionsService implements Resettable {
"arn:aws:states:::s3:listObjectsV2");
private static final Set<String> ITEM_READER_INPUT_TYPES = Set.of(
"MANIFEST", "JSON", "CSV", "JSONL", "PARQUET");
+ private static final String RESULT_WRITER_RESOURCE = "arn:aws:states:::s3:putObject";
+ private static final Set<String> RESULT_WRITER_TRANSFORMATIONS = Set.of("NONE", "COMPACT", "FLATTEN");
+ private static final Set<String> RESULT_WRITER_OUTPUT_TYPES = Set.of("JSON", "JSONL");
@Inject
public StepFunctionsService(StorageFactory storageFactory, RegionResolver regionResolver,
@@ -562,8 +565,13 @@ public class StepFunctionsService implements Resettable {
}
}
- if ("Map".equals(stateDef.path("Type").asText()) && stateDef.has("ItemReader")) {
- validateItemReader(stateName, stateDef, errors);
+ if ("Map".equals(stateDef.path("Type").asText())) {
+ if (stateDef.has("ItemReader")) {
+ validateItemReader(stateName, stateDef, errors);
+ }
+ if (stateDef.has("ResultWriter")) {
+ validateResultWriter(stateName, stateDef.get("ResultWriter"), stateIsJsonata, errors);
+ }
}
}
@@ -583,4 +591,95 @@ public class StepFunctionsService implements Resettable {
+ " at /States/" + stateName + "/ItemReader/ReaderConfig/InputType");
}
}
+
+ private void validateResultWriter(String stateName, JsonNode writer,
+ boolean jsonata, List<String> errors) {
+ String statePath = "/States/" + stateName;
+ String writerPath = statePath + "/ResultWriter";
+ if (!writer.isObject() || writer.isEmpty()) {
+ errors.add("The field 'ResultWriter' must specify WriterConfig or Resource with "
+ + (jsonata ? "Arguments" : "Parameters") + " at " + statePath);
+ return;
+ }
+
+ String destinationField = jsonata ? "Arguments" : "Parameters";
+ String unsupportedDestinationField = jsonata ? "Parameters" : "Arguments";
+ if (writer.has(unsupportedDestinationField)) {
+ errors.add("The field '" + unsupportedDestinationField + "' is not supported for the '"
+ + (jsonata ? "JSONata" : "JSONPath") + "' QueryLanguage at "
+ + writerPath + "/" + unsupportedDestinationField);
+ }
+ boolean hasConfig = writer.has("WriterConfig") && writer.get("WriterConfig").isObject();
+ boolean hasResource = writer.has("Resource");
+ boolean destinationIsExpression = jsonata
+ && writer.path(destinationField).isTextual()
+ && JsonataEvaluator.isExpression(writer.path(destinationField).asText());
+ boolean hasDestination = writer.has(destinationField)
+ && (writer.get(destinationField).isObject() || destinationIsExpression);
+
+ if (writer.has("WriterConfig") && !hasConfig) {
+ errors.add("The field 'WriterConfig' must be an object at " + writerPath);
+ }
+ if (writer.has(destinationField) && !hasDestination) {
+ errors.add("The field '" + destinationField + "' must be an object"
+ + (jsonata ? " or a JSONata expression" : "") + " at " + writerPath);
+ }
+ if ((!hasConfig && !hasResource && !hasDestination) || hasResource != hasDestination) {
+ errors.add("The field 'ResultWriter' must specify WriterConfig alone, Resource with "
+ + destinationField + ", or all three fields at " + statePath);
+ }
+
+ if (hasResource && (!writer.get("Resource").isTextual()
+ || !RESULT_WRITER_RESOURCE.equals(writer.get("Resource").asText()))) {
+ errors.add("The field 'Resource' does not match the allowed value "
+ + RESULT_WRITER_RESOURCE + " at " + writerPath);
+ }
+
+ if (hasDestination && !destinationIsExpression) {
+ JsonNode destination = writer.get(destinationField);
+ boolean hasBucket = destination.hasNonNull("Bucket")
+ || (!jsonata && destination.hasNonNull("Bucket.$"));
+ if (!hasBucket) {
+ errors.add("The field 'Bucket' is required at " + writerPath + "/" + destinationField);
+ } else if (destination.has("Bucket") && !destination.get("Bucket").isTextual()) {
+ errors.add("The field 'Bucket' must be a string at "
+ + writerPath + "/" + destinationField + "/Bucket");
+ } else if (!jsonata && destination.has("Bucket.$")
+ && !destination.get("Bucket.$").isTextual()) {
+ errors.add("The field 'Bucket.$' must be a string at "
+ + writerPath + "/" + destinationField + "/Bucket.$");
+ }
+ if (destination.has("Prefix") && !destination.get("Prefix").isTextual()) {
+ errors.add("The field 'Prefix' must be a string at "
+ + writerPath + "/" + destinationField + "/Prefix");
+ }
+ if (!jsonata && destination.has("Prefix.$")
+ && !destination.get("Prefix.$").isTextual()) {
+ errors.add("The field 'Prefix.$' must be a string at "
+ + writerPath + "/" + destinationField + "/Prefix.$");
+ }
+ }
+
+ if (hasConfig) {
+ JsonNode config = writer.get("WriterConfig");
+ JsonNode transformationNode = config.get("Transformation");
+ JsonNode outputTypeNode = config.get("OutputType");
+ if (transformationNode == null || !transformationNode.isTextual()
+ || outputTypeNode == null || !outputTypeNode.isTextual()) {
+ errors.add("The field 'WriterConfig' must specify string Transformation and OutputType at "
+ + writerPath + "/WriterConfig");
+ } else {
+ String transformation = transformationNode.asText();
+ if (!RESULT_WRITER_TRANSFORMATIONS.contains(transformation)) {
+ errors.add("The field 'Transformation' should have one of these values: "
+ + RESULT_WRITER_TRANSFORMATIONS + " at " + writerPath + "/WriterConfig");
+ }
+ String outputType = outputTypeNode.asText();
+ if (!RESULT_WRITER_OUTPUT_TYPES.contains(outputType)) {
+ errors.add("The field 'OutputType' should have one of these values: "
+ + RESULT_WRITER_OUTPUT_TYPES + " at " + writerPath + "/WriterConfig");
+ }
+ }
+ }
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorResultWriterTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorResultWriterTest.java
new file mode 100644
index 0000000000..06c2be1e51
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorResultWriterTest.java
@@ -0,0 +1,575 @@
+package io.github.hectorvent.floci.services.stepfunctions;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbJsonHandler;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;
+import io.github.hectorvent.floci.services.lambda.LambdaExecutorService;
+import io.github.hectorvent.floci.services.lambda.LambdaFunctionStore;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.sqs.SqsJsonHandler;
+import io.github.hectorvent.floci.services.stepfunctions.model.Execution;
+import io.github.hectorvent.floci.services.stepfunctions.model.HistoryEvent;
+import io.github.hectorvent.floci.services.stepfunctions.model.StateMachine;
+import jakarta.enterprise.inject.Instance;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit coverage for the Distributed Map {@code ResultWriter} emulation: the {@code WriterConfig}
+ * transformations ({@code NONE}/{@code COMPACT}/{@code FLATTEN}), the S3 export (manifest.json +
+ * result files), and the {@code {MapRunArn, ResultWriterDetails}} object the Map state returns —
+ * per the AWS Step Functions ResultWriter specification.
+ */
+class AslExecutorResultWriterTest {
+
+ private final ObjectMapper mapper = new ObjectMapper();
+ private S3Service s3Service;
+ private AslExecutor executor;
+ private StateMachine sm;
+ private JsonNode context;
+
+ @BeforeEach
+ @SuppressWarnings("unchecked")
+ void setUp() {
+ s3Service = mock(S3Service.class);
+ when(s3Service.putObject(anyString(), anyString(), any(byte[].class), anyString(), any()))
+ .thenReturn(null);
+ executor = new AslExecutor(
+ mock(LambdaExecutorService.class),
+ mock(LambdaFunctionStore.class),
+ mock(DynamoDbService.class),
+ mock(DynamoDbJsonHandler.class),
+ mock(SqsJsonHandler.class),
+ mock(io.github.hectorvent.floci.services.cloudformation.CloudFormationQueryHandler.class),
+ mock(io.github.hectorvent.floci.services.ec2.Ec2Service.class),
+ s3Service,
+ mock(io.github.hectorvent.floci.services.ecs.EcsService.class),
+ mock(io.github.hectorvent.floci.services.ecs.EcsJsonHandler.class),
+ mapper,
+ new JsonataEvaluator(mapper),
+ mock(Instance.class),
+ mock(EmulatorConfig.class),
+ null);
+
+ sm = new StateMachine();
+ sm.setStateMachineArn("arn:aws:states:us-east-1:000000000000:stateMachine:orderProcessing");
+ sm.setName("orderProcessing");
+
+ ObjectNode ctx = mapper.createObjectNode();
+ ctx.putObject("StateMachine").put("Name", "orderProcessing");
+ ctx.putObject("Execution").put("Name", "exec-1");
+ context = ctx;
+ }
+
+ private ArrayNode arr(String... jsons) throws Exception {
+ ArrayNode a = mapper.createArrayNode();
+ for (String j : jsons) {
+ a.add(mapper.readTree(j));
+ }
+ return a;
+ }
+
+ @Test
+ void exportWritesManifestAndResultFileAndReturnsResultWriterDetails() throws Exception {
+ JsonNode stateDef = mapper.readTree("""
+ {"Type":"Map",
+ "Label":"orders",
+ "ResultWriter":{"Resource":"arn:aws:states:::s3:putObject",
+ "Parameters":{"Bucket":"my-bucket","Prefix":"/csvJobs//"}}}
+ """);
+ ArrayNode results = arr("{\"ok\":1}", "{\"ok\":2}");
+ ArrayNode inputs = arr("{\"in\":1}", "{\"in\":2}");
+ List<long[]> timings = List.of(new long[]{1000L, 2000L}, new long[]{3000L, 4000L});
+
+ JsonNode out = executor.applyResultWriter("Process", stateDef, mapper.createObjectNode(),
+ results, inputs, timings, sm, context, false);
+
+ // The Map state returns the MapRunArn + the S3 manifest location, not the inline array.
+ assertTrue(out.has("MapRunArn"));
+ assertTrue(out.path("MapRunArn").asText().startsWith(
+ "arn:aws:states:us-east-1:000000000000:mapRun:orderProcessing/orders:"));
+ assertEquals("my-bucket", out.path("ResultWriterDetails").path("Bucket").asText());
+ String manifestKey = out.path("ResultWriterDetails").path("Key").asText();
+ assertTrue(manifestKey.startsWith("/csvJobs//"), manifestKey);
+ assertTrue(manifestKey.endsWith("/manifest.json"), manifestKey);
+ // The manifest uuid segment must match the MapRunArn uuid.
+ String mapRunUuid = out.path("MapRunArn").asText().substring(
+ out.path("MapRunArn").asText().lastIndexOf(':') + 1);
+ assertTrue(manifestKey.contains("/" + mapRunUuid + "/"), manifestKey);
+
+ // Capture the two S3 writes: SUCCEEDED_0.json and manifest.json.
+ ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<byte[]> bodies = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service, org.mockito.Mockito.times(2))
+ .putObject(eq("my-bucket"), keys.capture(), bodies.capture(), anyString(), any());
+
+ int manifestIdx = keys.getAllValues().indexOf(manifestKey);
+ assertTrue(manifestIdx >= 0, "manifest.json should have been written");
+ JsonNode manifest = mapper.readTree(bodies.getAllValues().get(manifestIdx));
+ assertEquals("my-bucket", manifest.path("DestinationBucket").asText());
+ assertEquals(out.path("MapRunArn").asText(), manifest.path("MapRunArn").asText());
+ assertEquals(0, manifest.path("ResultFiles").path("FAILED").size());
+ assertEquals(0, manifest.path("ResultFiles").path("PENDING").size());
+ assertEquals(1, manifest.path("ResultFiles").path("SUCCEEDED").size());
+ JsonNode succeededEntry = manifest.path("ResultFiles").path("SUCCEEDED").get(0);
+ assertTrue(succeededEntry.path("Key").asText().endsWith("SUCCEEDED_0.json"));
+ assertTrue(succeededEntry.path("Size").asInt() > 0);
+
+ // Default (no WriterConfig) export uses Transformation NONE: execution records with metadata.
+ int recIdx = manifestIdx == 0 ? 1 : 0;
+ JsonNode records = mapper.readTree(bodies.getAllValues().get(recIdx));
+ assertTrue(records.isArray());
+ assertEquals(2, records.size());
+ JsonNode rec = records.get(0);
+ assertEquals("SUCCEEDED", rec.path("Status").asText());
+ assertEquals("NOT_REDRIVABLE", rec.path("RedriveStatus").asText());
+ assertTrue(rec.path("ExecutionArn").asText().contains(":execution:orderProcessing/orders:"));
+ assertEquals("arn:aws:states:us-east-1:000000000000:stateMachine:orderProcessing/orders",
+ rec.path("StateMachineArn").asText());
+ // Output is stored as a JSON string (not a nested object), matching AWS.
+ assertTrue(rec.path("Output").isTextual());
+ assertEquals("{\"ok\":1}", rec.path("Output").asText());
+ assertTrue(rec.path("InputDetails").path("Included").asBoolean());
+ }
+
+ @Test
+ void compactTransformationReturnsRawOutputsAndDoesNotWriteWhenNoResource() throws Exception {
+ JsonNode stateDef = mapper.readTree("""
+ {"Type":"Map",
+ "ResultWriter":{"WriterConfig":{"Transformation":"COMPACT","OutputType":"JSON"}}}
+ """);
+ ArrayNode results = arr("[{\"a\":1}]", "[{\"b\":2}]");
+
+ JsonNode out = executor.applyResultWriter("Process", stateDef, mapper.createObjectNode(),
+ results, mapper.createArrayNode(), List.of(), sm, context, false);
+
+ // COMPACT keeps the original per-child array structure; no S3 Resource -> no export.
+ assertEquals(results, out);
+ verify(s3Service, org.mockito.Mockito.never())
+ .putObject(anyString(), anyString(), any(byte[].class), anyString(), any());
+ }
+
+ @Test
+ void flattenTransformationFlattensChildArrays() throws Exception {
+ JsonNode stateDef = mapper.readTree("""
+ {"Type":"Map",
+ "ResultWriter":{"WriterConfig":{"Transformation":"FLATTEN","OutputType":"JSON"}}}
+ """);
+ ArrayNode results = arr("[{\"id\":1},{\"id\":2}]", "[{\"id\":3}]");
+
+ JsonNode out = executor.applyResultWriter("Process", stateDef, mapper.createObjectNode(),
+ results, mapper.createArrayNode(), List.of(), sm, context, false);
+
+ assertTrue(out.isArray());
+ assertEquals(3, out.size());
+ assertEquals(1, out.get(0).path("id").asInt());
+ assertEquals(3, out.get(2).path("id").asInt());
+ }
+
+ @Test
+ void exportRecordsScalarStringInputAndOutputAsQuotedJson() throws Exception {
+ JsonNode stateDef = mapper.readTree("""
+ {"Type":"Map",
+ "ResultWriter":{"Resource":"arn:aws:states:::s3:putObject",
+ "Parameters":{"Bucket":"my-bucket","Prefix":"p"}}}
+ """);
+ // A child whose input and output are JSON string scalars.
+ ArrayNode results = arr("\"hello\"");
+ ArrayNode inputs = arr("\"world\"");
+ List<long[]> timings = List.of(new long[]{1000L, 2000L});
+
+ executor.applyResultWriter("Process", stateDef, mapper.createObjectNode(),
+ results, inputs, timings, sm, context, false);
+
+ ArgumentCaptor<byte[]> bodies = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service, org.mockito.Mockito.times(2))
+ .putObject(eq("my-bucket"), anyString(), bodies.capture(), anyString(), any());
+ // The records file is the JSON array write (the other write is manifest.json, an object).
+ JsonNode first = mapper.readTree(bodies.getAllValues().get(0));
+ JsonNode records = first.isArray() ? first : mapper.readTree(bodies.getAllValues().get(1));
+ JsonNode rec = records.get(0);
+ // AWS stores a child's Input/Output as a JSON-encoded string, so a string scalar keeps its
+ // surrounding quotes (regression: asText() would drop them and yield invalid JSON).
+ assertEquals("\"hello\"", rec.path("Output").asText());
+ assertEquals("\"world\"", rec.path("Input").asText());
+ }
+
+ @Test
+ void distributedMapAppliesInputPathBeforeItemsAndWriterParameters() throws Exception {
+ StateMachine machine = new StateMachine();
+ machine.setName("orderProcessing");
+ machine.setStateMachineArn("arn:aws:states:us-east-1:000000000000:stateMachine:orderProcessing");
+ machine.setRoleArn("arn:aws:iam::000000000000:role/test-role");
+ machine.setDefinition("""
+ {
+ "StartAt":"Process",
+ "States":{
+ "Process":{
+ "Type":"Map",
+ "Label":"orders",
+ "InputPath":"$.selected",
+ "ItemsPath":"$.items",
+ "ItemProcessor":{
+ "ProcessorConfig":{"Mode":"DISTRIBUTED","ExecutionType":"STANDARD"},
+ "StartAt":"PassItem",
+ "States":{"PassItem":{"Type":"Pass","End":true}}
+ },
+ "ResultWriter":{
+ "Resource":"arn:aws:states:::s3:putObject",
+ "Parameters":{
+ "Bucket.$":"$.destination.bucket",
+ "Prefix.$":"$.destination.prefix"
+ }
+ },
+ "End":true
+ }
+ }
+ }
+ """);
+ Execution execution = execution(machine, "input-path", """
+ {
+ "items":["wrong"],
+ "destination":{"bucket":"raw-bucket","prefix":"raw-prefix"},
+ "selected":{
+ "items":[{"id":1},{"id":2}],
+ "destination":{"bucket":"selected-bucket","prefix":"/csvJobs//"}
+ }
+ }
+ """);
+
+ executor.executeSync(machine, execution, new ArrayList<>(), (updated, history) -> { });
+
+ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());
+ JsonNode output = mapper.readTree(execution.getOutput());
+ assertEquals("selected-bucket", output.path("ResultWriterDetails").path("Bucket").asText());
+ String mapRunArn = output.path("MapRunArn").asText();
+ assertTrue(mapRunArn.startsWith(
+ "arn:aws:states:us-east-1:000000000000:mapRun:orderProcessing/orders:"), mapRunArn);
+ String mapRunId = mapRunArn.substring(mapRunArn.lastIndexOf(':') + 1);
+ String manifestKey = output.path("ResultWriterDetails").path("Key").asText();
+ assertTrue(manifestKey.startsWith("/csvJobs//" + mapRunId + "/"), manifestKey);
+
+ ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);
+ verify(s3Service, org.mockito.Mockito.times(2))
+ .putObject(eq("selected-bucket"), keys.capture(), any(byte[].class), anyString(), any());
+ assertTrue(keys.getAllValues().stream().allMatch(key -> key.startsWith("/csvJobs//" + mapRunId + "/")));
+ }
+
+ @Test
+ void jsonataArgumentsExpressionCanUseWorkflowVariables() throws Exception {
+ StateMachine machine = new StateMachine();
+ machine.setName("orderProcessing");
+ machine.setStateMachineArn("arn:aws:states:us-east-1:000000000000:stateMachine:orderProcessing");
+ machine.setRoleArn("arn:aws:iam::000000000000:role/test-role");
+ machine.setDefinition("""
+ {
+ "QueryLanguage":"JSONata",
+ "StartAt":"SetDestination",
+ "States":{
+ "SetDestination":{
+ "Type":"Pass",
+ "Assign":{"destination":"{% $states.input.destination %}"},
+ "Next":"Process"
+ },
+ "Process":{
+ "Type":"Map",
+ "Label":"orders",
+ "Items":"{% $states.input.items %}",
+ "ItemProcessor":{
+ "ProcessorConfig":{"Mode":"DISTRIBUTED","ExecutionType":"STANDARD"},
+ "StartAt":"PassItem",
+ "States":{"PassItem":{"Type":"Pass","End":true}}
+ },
+ "ResultWriter":{
+ "Resource":"arn:aws:states:::s3:putObject",
+ "Arguments":"{% $destination %}"
+ },
+ "End":true
+ }
+ }
+ }
+ """);
+ Execution execution = execution(machine, "jsonata-writer", """
+ {"items":[{"id":1}],"destination":{"Bucket":"jsonata-bucket","Prefix":"exports"}}
+ """);
+
+ executor.executeSync(machine, execution, new ArrayList<>(), (updated, history) -> { });
+
+ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());
+ JsonNode output = mapper.readTree(execution.getOutput());
+ assertEquals("jsonata-bucket", output.path("ResultWriterDetails").path("Bucket").asText());
+ ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);
+ verify(s3Service, org.mockito.Mockito.times(2))
+ .putObject(eq("jsonata-bucket"), keys.capture(), any(byte[].class), anyString(), any());
+ assertTrue(keys.getAllValues().stream().allMatch(key -> key.startsWith("exports/")));
+ }
+
+ @Test
+ void jsonataArgumentsExpressionMustResolveToAnObject() throws Exception {
+ JsonNode stateDef = mapper.readTree("""
+ {"Type":"Map",
+ "ResultWriter":{"Resource":"arn:aws:states:::s3:putObject",
+ "Arguments":"{% 1 %}"}}
+ """);
+
+ AslExecutor.FailStateException failure = assertThrows(AslExecutor.FailStateException.class,
+ () -> executor.applyResultWriter("Process", stateDef, mapper.createObjectNode(),
+ arr("{\"ok\":true}"), mapper.createArrayNode(), List.of(),
+ sm, context, true));
+
+ assertEquals("States.QueryEvaluationError", failure.error);
+ assertTrue(failure.getMessage().contains("must resolve to an object"), failure.getMessage());
+ verify(s3Service, org.mockito.Mockito.never())
+ .putObject(anyString(), anyString(), any(byte[].class), anyString(), any());
+ }
+
+ @Test
+ void jsonataDestinationFieldsMustResolveToStrings() throws Exception {
+ JsonNode stateDef = mapper.readTree("""
+ {"Type":"Map",
+ "ResultWriter":{"Resource":"arn:aws:states:::s3:putObject",
+ "Arguments":{"Bucket":"{% 42 %}","Prefix":"results"}}}
+ """);
+
+ AslExecutor.FailStateException failure = assertThrows(AslExecutor.FailStateException.class,
+ () -> executor.applyResultWriter("Process", stateDef, mapper.createObjectNode(),
+ arr("{\"ok\":true}"), mapper.createArrayNode(), List.of(),
+ sm, context, true));
+
+ assertEquals("States.QueryEvaluationError", failure.error);
+ assertTrue(failure.getMessage().contains("Bucket must resolve to a string"), failure.getMessage());
+ verify(s3Service, org.mockito.Mockito.never())
+ .putObject(anyString(), anyString(), any(byte[].class), anyString(), any());
+ }
+
+ @Test
+ void jsonataDestinationNullsAreTypeErrorsRatherThanMissingValues() throws Exception {
+ JsonNode stateDef = mapper.readTree("""
+ {"Type":"Map",
+ "ResultWriter":{"Resource":"arn:aws:states:::s3:putObject",
+ "Arguments":"{% $states.input.destination %}"}}
+ """);
+
+ AslExecutor.FailStateException bucketFailure = assertThrows(AslExecutor.FailStateException.class,
+ () -> executor.applyResultWriter("Process", stateDef,
+ mapper.readTree("{\"destination\":{\"Bucket\":null}}"),
+ arr("{\"ok\":true}"), mapper.createArrayNode(), List.of(),
+ sm, context, true));
+ assertEquals("States.QueryEvaluationError", bucketFailure.error);
+ assertTrue(bucketFailure.getMessage().contains("Bucket must resolve to a string"));
+
+ AslExecutor.FailStateException prefixFailure = assertThrows(AslExecutor.FailStateException.class,
+ () -> executor.applyResultWriter("Process", stateDef,
+ mapper.readTree("{\"destination\":{\"Bucket\":\"b\",\"Prefix\":null}}"),
+ arr("{\"ok\":true}"), mapper.createArrayNode(), List.of(),
+ sm, context, true));
+ assertEquals("States.QueryEvaluationError", prefixFailure.error);
+ assertTrue(prefixFailure.getMessage().contains("Prefix must resolve to a string"));
+
+ JsonNode nestedBucketStateDef = mapper.readTree("""
+ {"Type":"Map",
+ "ResultWriter":{"Resource":"arn:aws:states:::s3:putObject",
+ "Arguments":{"Bucket":"{% null %}","Prefix":"results"}}}
+ """);
+ AslExecutor.FailStateException nestedBucketFailure = assertThrows(
+ AslExecutor.FailStateException.class,
+ () -> executor.applyResultWriter("Process", nestedBucketStateDef, mapper.createObjectNode(),
+ arr("{\"ok\":true}"), mapper.createArrayNode(), List.of(),
+ sm, context, true));
+ assertEquals("States.QueryEvaluationError", nestedBucketFailure.error);
+ assertTrue(nestedBucketFailure.getMessage().contains("Bucket must resolve to a string"));
+
+ JsonNode nestedPrefixStateDef = mapper.readTree("""
+ {"Type":"Map",
+ "ResultWriter":{"Resource":"arn:aws:states:::s3:putObject",
+ "Arguments":{"Bucket":"b","Prefix":"{% $states.input.missing %}"}}}
+ """);
+ AslExecutor.FailStateException nestedPrefixFailure = assertThrows(
+ AslExecutor.FailStateException.class,
+ () -> executor.applyResultWriter("Process", nestedPrefixStateDef, mapper.createObjectNode(),
+ arr("{\"ok\":true}"), mapper.createArrayNode(), List.of(),
+ sm, context, true));
+ assertEquals("States.QueryEvaluationError", nestedPrefixFailure.error);
+ assertTrue(nestedPrefixFailure.getMessage().contains("Prefix must resolve to a string"));
+
+ verify(s3Service, org.mockito.Mockito.never())
+ .putObject(anyString(), anyString(), any(byte[].class), anyString(), any());
+ }
+
+ @Test
+ void crossRegionDestinationRaisesResultWriterFailedBeforeWriting() throws Exception {
+ when(s3Service.getBucketRegion("other-region-bucket")).thenReturn("us-west-2");
+ JsonNode stateDef = mapper.readTree("""
+ {"Type":"Map",
+ "ResultWriter":{"Resource":"arn:aws:states:::s3:putObject",
+ "Parameters":{"Bucket":"other-region-bucket","Prefix":"results"}}}
+ """);
+
+ AslExecutor.FailStateException failure = assertThrows(AslExecutor.FailStateException.class,
+ () -> executor.applyResultWriter("Process", stateDef, mapper.createObjectNode(),
+ arr("{\"ok\":true}"), mapper.createArrayNode(), List.of(),
+ sm, context, false));
+
+ assertEquals("States.ResultWriterFailed", failure.error);
+ assertTrue(failure.getMessage().contains("same AWS Region"), failure.getMessage());
+ verify(s3Service, org.mockito.Mockito.never())
+ .putObject(anyString(), anyString(), any(byte[].class), anyString(), any());
+ }
+
+ @Test
+ void s3WriteFailureRaisesCatchableResultWriterFailed() throws Exception {
+ when(s3Service.putObject(anyString(), anyString(), any(byte[].class), anyString(), any()))
+ .thenThrow(new AwsException("NoSuchBucket", "Destination bucket does not exist", 404));
+ StateMachine machine = new StateMachine();
+ machine.setName("orderProcessing");
+ machine.setStateMachineArn("arn:aws:states:us-east-1:000000000000:stateMachine:orderProcessing");
+ machine.setRoleArn("arn:aws:iam::000000000000:role/test-role");
+ machine.setDefinition("""
+ {
+ "StartAt":"Process",
+ "States":{
+ "Process":{
+ "Type":"Map",
+ "ItemsPath":"$.items",
+ "ItemProcessor":{
+ "ProcessorConfig":{"Mode":"DISTRIBUTED","ExecutionType":"STANDARD"},
+ "StartAt":"PassItem",
+ "States":{"PassItem":{"Type":"Pass","End":true}}
+ },
+ "ResultWriter":{
+ "Resource":"arn:aws:states:::s3:putObject",
+ "Parameters":{"Bucket":"missing-bucket","Prefix":"results"}
+ },
+ "Catch":[{
+ "ErrorEquals":["States.ResultWriterFailed"],
+ "ResultPath":"$.writerError",
+ "Next":"Recovered"
+ }],
+ "End":true
+ },
+ "Recovered":{"Type":"Pass","End":true}
+ }
+ }
+ """);
+ Execution execution = execution(machine, "writer-failure", "{\"items\":[1]}");
+
+ executor.executeSync(machine, execution, new ArrayList<>(), (updated, history) -> { });
+
+ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());
+ JsonNode output = mapper.readTree(execution.getOutput());
+ assertEquals("States.ResultWriterFailed",
+ output.path("writerError").path("Error").asText());
+ assertTrue(output.path("writerError").path("Cause").asText()
+ .contains("Destination bucket does not exist"), output.toString());
+ }
+
+ @Test
+ void exportWithoutResolvedBucketFailsInsteadOfReturningInlineResults() throws Exception {
+ JsonNode stateDef = mapper.readTree("""
+ {"Type":"Map",
+ "ResultWriter":{"Resource":"arn:aws:states:::s3:putObject",
+ "Parameters":{"Prefix":"results"}}}
+ """);
+
+ AslExecutor.FailStateException failure = assertThrows(AslExecutor.FailStateException.class,
+ () -> executor.applyResultWriter("Process", stateDef, mapper.createObjectNode(),
+ arr("{\"ok\":true}"), mapper.createArrayNode(), List.of(),
+ sm, context, false));
+
+ assertEquals("States.ResultWriterFailed", failure.error);
+ verify(s3Service, org.mockito.Mockito.never())
+ .putObject(anyString(), anyString(), any(byte[].class), anyString(), any());
+ }
+
+ @Test
+ void resultWriterOnInlineMapIsRejected() {
+ StateMachine machine = new StateMachine();
+ machine.setName("orderProcessing");
+ machine.setStateMachineArn("arn:aws:states:us-east-1:000000000000:stateMachine:orderProcessing");
+ machine.setRoleArn("arn:aws:iam::000000000000:role/test-role");
+ // An INLINE map (no ProcessorConfig Mode) carrying a ResultWriter but no ItemReader.
+ machine.setDefinition("""
+ {"StartAt":"M","States":{"M":{"Type":"Map",
+ "ItemProcessor":{"StartAt":"P","States":{"P":{"Type":"Pass","End":true}}},
+ "ResultWriter":{"Resource":"arn:aws:states:::s3:putObject",
+ "Parameters":{"Bucket":"b","Prefix":"p"}},
+ "End":true}}}
+ """);
+ Execution execution = new Execution();
+ execution.setName("exec-1");
+ execution.setExecutionArn(
+ "arn:aws:states:us-east-1:000000000000:execution:orderProcessing:exec-1");
+ execution.setStateMachineArn(machine.getStateMachineArn());
+ execution.setInput("[1,2]");
+
+ executor.executeSync(machine, execution, new ArrayList<HistoryEvent>(), (u, e) -> { });
+
+ // AWS rejects ResultWriter on an inline map; Floci fails the execution rather than exporting.
+ assertEquals("FAILED", execution.getStatus());
+ assertEquals("States.Runtime", execution.getError());
+ assertTrue(execution.getCause().contains("not supported for INLINE maps"), execution.getCause());
+ verify(s3Service, org.mockito.Mockito.never())
+ .putObject(anyString(), anyString(), any(byte[].class), anyString(), any());
+ }
+
+ @Test
+ void jsonlOutputTypeWritesOneJsonRecordPerLineInJsonNamedFile() throws Exception {
+ JsonNode stateDef = mapper.readTree("""
+ {"Type":"Map",
+ "ResultWriter":{"Resource":"arn:aws:states:::s3:putObject",
+ "WriterConfig":{"Transformation":"COMPACT","OutputType":"JSONL"},
+ "Parameters":{"Bucket":"b","Prefix":"p"}}}
+ """);
+ ArrayNode results = arr("{\"a\":1}", "{\"a\":2}");
+
+ JsonNode out = executor.applyResultWriter("Process", stateDef, mapper.createObjectNode(),
+ results, mapper.createArrayNode(), List.of(), sm, context, false);
+
+ String resultKey = null;
+ ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<byte[]> bodies = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service, org.mockito.Mockito.times(2))
+ .putObject(anyString(), keys.capture(), bodies.capture(), anyString(), any());
+ for (int i = 0; i < keys.getAllValues().size(); i++) {
+ if (keys.getAllValues().get(i).endsWith("SUCCEEDED_0.json")) {
+ resultKey = keys.getAllValues().get(i);
+ String body = new String(bodies.getAllValues().get(i), java.nio.charset.StandardCharsets.UTF_8);
+ String[] lines = body.strip().split("\n");
+ assertEquals(2, lines.length);
+ assertEquals(1, mapper.readTree(lines[0]).path("a").asInt());
+ }
+ }
+ assertFalse(resultKey == null, "JSONL serialization should retain AWS's .json result-file name");
+ assertTrue(out.path("ResultWriterDetails").path("Key").asText().endsWith("/manifest.json"));
+ }
+
+ private Execution execution(StateMachine machine, String name, String input) {
+ Execution execution = new Execution();
+ execution.setName(name);
+ execution.setExecutionArn(
+ "arn:aws:states:us-east-1:000000000000:execution:" + machine.getName() + ":" + name);
+ execution.setStateMachineArn(machine.getStateMachineArn());
+ execution.setInput(input);
+ return execution;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java
index 8f9ae51359..d2212369f9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.stepfunctions;
+import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
import io.restassured.response.Response;
@@ -27,6 +28,7 @@ class StepFunctionsValidateStateMachineDefinitionIntegrationTest {
private static final String CT = "application/x-amz-json-1.0";
private static final String TARGET = "AWSStepFunctions.ValidateStateMachineDefinition";
private static final String LIST_TARGET = "AWSStepFunctions.ListStateMachines";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
// ASL with the inner double-quotes already JSON-escaped, so it embeds cleanly
// inside the outer JSON request body as the value of "definition".
@@ -90,6 +92,56 @@ class StepFunctionsValidateStateMachineDefinitionIntegrationTest {
RestAssuredJsonUtils.configureAwsContentTypes();
}
+ private static Response validateDefinition(String definition) {
+ String body = OBJECT_MAPPER.createObjectNode().put("definition", definition).toString();
+ return given().contentType(CT).header("X-Amz-Target", TARGET)
+ .body(body)
+ .when().post("/");
+ }
+
+ private static String distributedMapWithResultWriter(String resultWriter) {
+ return """
+ {
+ "StartAt":"ProcessItems",
+ "States":{
+ "ProcessItems":{
+ "Type":"Map",
+ "ItemsPath":"$.items",
+ "ItemProcessor":{
+ "ProcessorConfig":{"Mode":"DISTRIBUTED","ExecutionType":"STANDARD"},
+ "StartAt":"PassItem",
+ "States":{"PassItem":{"Type":"Pass","End":true}}
+ },
+ "ResultWriter":__RESULT_WRITER__,
+ "End":true
+ }
+ }
+ }
+ """.replace("__RESULT_WRITER__", resultWriter);
+ }
+
+ private static String jsonataDistributedMapWithResultWriter(String resultWriter) {
+ return """
+ {
+ "QueryLanguage":"JSONata",
+ "StartAt":"ProcessItems",
+ "States":{
+ "ProcessItems":{
+ "Type":"Map",
+ "Items":"{% $states.input.items %}",
+ "ItemProcessor":{
+ "ProcessorConfig":{"Mode":"DISTRIBUTED","ExecutionType":"STANDARD"},
+ "StartAt":"PassItem",
+ "States":{"PassItem":{"Type":"Pass","End":true}}
+ },
+ "ResultWriter":__RESULT_WRITER__,
+ "End":true
+ }
+ }
+ }
+ """.replace("__RESULT_WRITER__", resultWriter);
+ }
+
@Test
void validDefinition_returnsOK() {
given().contentType(CT).header("X-Amz-Target", TARGET)
@@ -158,6 +210,172 @@ class StepFunctionsValidateStateMachineDefinitionIntegrationTest {
.body("diagnostics[0].location", equalTo("/States/ProcessItems/ItemReader/ReaderConfig/InputType"));
}
+ @Test
+ void validResultWriterResourceAndParameters_returnsOK() {
+ String definition = distributedMapWithResultWriter("""
+ {
+ "Resource":"arn:aws:states:::s3:putObject",
+ "WriterConfig":{"Transformation":"FLATTEN","OutputType":"JSONL"},
+ "Parameters":{"Bucket.$":"$.destination.bucket","Prefix":"results"}
+ }
+ """);
+
+ validateDefinition(definition)
+ .then().statusCode(200)
+ .body("result", equalTo("OK"))
+ .body("diagnostics", hasSize(0));
+ }
+
+ @Test
+ void jsonataResultWriterAcceptsExpressionFormArguments() {
+ String definition = jsonataDistributedMapWithResultWriter("""
+ {
+ "Resource":"arn:aws:states:::s3:putObject",
+ "Arguments":"{% $states.input.destination %}"
+ }
+ """);
+
+ validateDefinition(definition)
+ .then().statusCode(200)
+ .body("result", equalTo("OK"))
+ .body("diagnostics", hasSize(0));
+ }
+
+ @Test
+ void jsonataResultWriterRejectsJsonpathParameters() {
+ String definition = jsonataDistributedMapWithResultWriter("""
+ {
+ "Resource":"arn:aws:states:::s3:putObject",
+ "Arguments":{"Bucket":"results-bucket"},
+ "Parameters":{"Bucket":"ignored-bucket"}
+ }
+ """);
+
+ validateDefinition(definition)
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].location", equalTo(
+ "/States/ProcessItems/ResultWriter/Parameters"));
+ }
+
+ @Test
+ void jsonpathResultWriterRejectsJsonataArguments() {
+ String definition = distributedMapWithResultWriter("""
+ {
+ "Resource":"arn:aws:states:::s3:putObject",
+ "Parameters":{"Bucket":"results-bucket"},
+ "Arguments":{"Bucket":"ignored-bucket"}
+ }
+ """);
+
+ validateDefinition(definition)
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].location", equalTo(
+ "/States/ProcessItems/ResultWriter/Arguments"));
+ }
+
+ @Test
+ void writerConfigRequiresTransformationAndOutputTypeTogether() {
+ String definition = distributedMapWithResultWriter("""
+ {"WriterConfig":{"Transformation":"COMPACT"}}
+ """);
+
+ validateDefinition(definition)
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].location", equalTo(
+ "/States/ProcessItems/ResultWriter/WriterConfig"));
+ }
+
+ @Test
+ void writerConfigValuesMustBeStrings() {
+ String definition = distributedMapWithResultWriter("""
+ {"WriterConfig":{"Transformation":{},"OutputType":[]}}
+ """);
+
+ validateDefinition(definition)
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].location", equalTo(
+ "/States/ProcessItems/ResultWriter/WriterConfig"));
+ }
+
+ @Test
+ void resultWriterDestinationFieldsMustBeStrings() {
+ String definition = distributedMapWithResultWriter("""
+ {
+ "Resource":"arn:aws:states:::s3:putObject",
+ "Parameters":{"Bucket":42,"Prefix":{}}
+ }
+ """);
+
+ validateDefinition(definition)
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(2));
+ }
+
+ @Test
+ void emptyResultWriter_returnsFailWithSchemaError() {
+ validateDefinition(distributedMapWithResultWriter("{}"))
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].location", equalTo("/States/ProcessItems/ResultWriter"));
+ }
+
+ @Test
+ void resultWriterResourceWithoutParameters_returnsFailWithSchemaError() {
+ String definition = distributedMapWithResultWriter("""
+ {"Resource":"arn:aws:states:::s3:putObject"}
+ """);
+
+ validateDefinition(definition)
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].location", equalTo("/States/ProcessItems/ResultWriter"));
+ }
+
+ @Test
+ void nullResultWriterResourceIsNotTreatedAsAbsent() {
+ String definition = distributedMapWithResultWriter("""
+ {
+ "Resource":null,
+ "Parameters":{"Bucket":"results-bucket"},
+ "WriterConfig":{"Transformation":"COMPACT","OutputType":"JSON"}
+ }
+ """);
+
+ validateDefinition(definition)
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].location", equalTo(
+ "/States/ProcessItems/ResultWriter/Resource"));
+ }
+
+ @Test
+ void unsupportedResultWriterResource_returnsFailWithSchemaError() {
+ String definition = distributedMapWithResultWriter("""
+ {
+ "Resource":"arn:aws:states:::s3:unknownOperation",
+ "Parameters":{"Bucket":"results-bucket"}
+ }
+ """);
+
+ validateDefinition(definition)
+ .then().statusCode(200)
+ .body("result", equalTo("FAIL"))
+ .body("diagnostics", hasSize(1))
+ .body("diagnostics[0].location", equalTo("/States/ProcessItems/ResultWriter/Resource"));
+ }
+
@Test
void itemReaderWithoutDistributedMode_isAcceptedAtDefinitionTime() {
given().contentType(CT).header("X-Amz-Target", TARGET)