Light, fluffy, and always free - The AWS Local Emulator alternative
fix(stepfunctions): reject reserved API headers in definitions (#6082)
Andre Kurait2c6f2118e8c2
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.mdindex e3aa704a8d..5b9f7eaa9c 100644--- a/docs/services/step-functions.md+++ b/docs/services/step-functions.md@@ -804,6 +804,9 @@ in every partition. State machine and execution ARNs follow the region they are`arn:...:states:::apigateway:invoke` supports regional, same-account REST APIs with `POST`,`IAM_ROLE`, a named stage and resource path, and an object request body without null values.Headers accept multiple string values and HTTP token names. AWS-reserved headers are rejected.+Literal reserved names, including `Expect`, `Upgrade`, `Authorization` and `Host`, and the+`X-Amz-`, `X-Amzn-` and `X-Forwarded-` prefixes are rejected case-insensitively when a+definition is created, updated or validated, including JSONata arguments and nested workflows.An explicit `Content-Length` must equal the serialized body size; the HTTP client writes that header.Private endpoints, resource-policy authentication, query parameters and `AllowNullValues` remain unsupported.diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/ApiGatewayTaskInvoker.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/ApiGatewayTaskInvoker.javaindex 3199a9e4e0..1814a5ae81 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/ApiGatewayTaskInvoker.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/ApiGatewayTaskInvoker.java@@ -40,6 +40,7 @@ import java.util.Base64;import java.util.List;import java.util.Locale;import java.util.Map;+import java.util.Set;import java.util.TreeMap;import java.util.UUID;import java.util.stream.Collectors;@@ -48,6 +49,10 @@ import java.util.stream.Collectors;@ApplicationScopedpublic class ApiGatewayTaskInvoker {private static final SecureRandom RANDOM = new SecureRandom();+ private static final Set<String> RESERVED_HEADERS = Set.of(+ "authorization", "connection", "content-md5", "expect", "host", "max-forwards",+ "proxy-authenticate", "server", "te", "transfer-encoding", "trailer", "upgrade", "via",+ "www-authenticate");private static final DateTimeFormatter AMZ_DATE =DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC);private final ApiGatewayService apiGateway;@@ -230,10 +235,7 @@ public class ApiGatewayTaskInvoker {input.properties().forEach(entry -> {String name = entry.getKey().toLowerCase(Locale.ROOT);JsonNode values = entry.getValue();- if (!entry.getKey().matches("[!#$%&'*+.^_`|~0-9A-Za-z-]+") || name.startsWith("x-amz") || name.startsWith("x-forwarded")- || List.of("authorization", "connection", "content-md5", "expect", "host", "max-forwards",- "proxy-authenticate", "server", "te", "transfer-encoding", "trailer", "upgrade", "via",- "www-authenticate").contains(name)+ if (!entry.getKey().matches("[!#$%&'*+.^_`|~0-9A-Za-z-]+") || isReservedDefinitionHeader(name)|| !values.isArray() || values.isEmpty() || result.containsKey(name)) {throw unsupported("header name or value is not supported");}@@ -249,6 +251,11 @@ public class ApiGatewayTaskInvoker {return result;}+ static boolean isReservedDefinitionHeader(String lowerName) {+ return RESERVED_HEADERS.contains(lowerName) || lowerName.startsWith("x-amz-")+ || lowerName.startsWith("x-amzn-") || lowerName.startsWith("x-forwarded-");+ }+private static void sign(TreeMap<String, List<String>> headers, URI uri, byte[] payload, String region,String accessKey, String secret, String token) throws Exception {String stamp = AMZ_DATE.format(Instant.now());diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.javaindex 6fd1395a80..3bde8896d4 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsService.java@@ -2378,6 +2378,9 @@ public class StepFunctionsService implements Resettable, ResourceProvider {validateFieldsAllowedForType(statePath, stateDef, stateType, errors);validateTransitionTargets(statePath, stateDef, siblingStateNames, errors);+ if ("Task".equals(stateType)) {+ validateApiGatewayHeaders(statePath, stateDef, stateIsJsonata, errors);+ }// Measured on AWS: the downgrade is the whole answer for that state, and the fields its// refused language forbids are not named on top of it. Only this check is skipped; every@@ -2429,6 +2432,30 @@ public class StepFunctionsService implements Resettable, ResourceProvider {}}+ private static void validateApiGatewayHeaders(String statePath, JsonNode stateDef,+ boolean jsonata, List<String> errors) {+ if (StatesIntegration.parse(stateDef.path("Resource").asText(null))+ .filter(integration -> integration.isAnySuffix("apigateway", "invoke")).isEmpty()) {+ return;+ }+ String field = jsonata ? "Arguments" : "Parameters";+ JsonNode headers = stateDef.path(field).path("Headers");+ if (!headers.isObject()) {+ return;+ }+ List<String> unsupported = new ArrayList<>();+ headers.fieldNames().forEachRemaining(name -> {+ String lower = name.toLowerCase(Locale.ROOT);+ if (ApiGatewayTaskInvoker.isReservedDefinitionHeader(lower)) {+ unsupported.add(lower);+ }+ });+ if (!unsupported.isEmpty()) {+ errors.add(EXPLICIT_LOCATION_MARKER + "The 'Headers' field contains unsupported values: "+ + unsupported + MARKER_PAYLOAD_SEPARATOR + statePath + "/" + field);+ }+ }+/*** Reports every JSONata expression in the state that does not parse, or that reads the* top-level context, or that reads {@code $states.errorOutput} outside the one place itdiff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/ApiGatewayTaskInvokerTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/ApiGatewayTaskInvokerTest.javaindex f5f63e50b5..d0009634b5 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/ApiGatewayTaskInvokerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/ApiGatewayTaskInvokerTest.java@@ -24,6 +24,9 @@ import jakarta.enterprise.inject.Instance;import org.junit.jupiter.api.AfterEach;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;+import org.junit.jupiter.params.provider.ValueSource;import java.net.InetSocketAddress;import java.net.URI;@@ -64,6 +67,7 @@ class ApiGatewayTaskInvokerTest {private final AtomicReference<Throwable> serverFailure = new AtomicReference<>();private final AtomicInteger calls = new AtomicInteger();private final AtomicInteger status = new AtomicInteger(200);+ private final AtomicReference<String> responseBody = new AtomicReference<>("{\"workflow_id\":\"owned\"}");private ApiGatewayTaskInvoker invoker;// Quarkus and the JDK test server share the HttpServer simple name.private final io.quarkus.vertx.http.HttpServer listener = mock(io.quarkus.vertx.http.HttpServer.class);@@ -74,6 +78,7 @@ class ApiGatewayTaskInvokerTest {private final EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);private final Instance<ScpProvider> scps = mock(Instance.class);private final AtomicReference<List<String>> receivedHeaders = new AtomicReference<>();+ private final AtomicReference<String> observedHeader = new AtomicReference<>("x_test");@BeforeEachvoid setup() throws Exception {@@ -111,7 +116,7 @@ class ApiGatewayTaskInvokerTest {try {byte[] body = exchange.getRequestBody().readAllBytes();received.set(mapper.readTree(body));- receivedHeaders.set(exchange.getRequestHeaders().get("x_test"));+ receivedHeaders.set(exchange.getRequestHeaders().get(observedHeader.get()));assertEquals("/restapis/owned/gamma/_user_request_/create-workflow-execution", exchange.getRequestURI().getPath());assertEquals("owned-key", exchange.getRequestHeaders().getFirst("x-api-key"));SigV4AuthorizationHeader auth = SigV4AuthorizationHeader.parse(exchange.getRequestHeaders().getFirst("Authorization"));@@ -127,7 +132,7 @@ class ApiGatewayTaskInvokerTest {} catch (Throwable error) {serverFailure.set(error);}- byte[] body = "{\"workflow_id\":\"owned\"}".getBytes(StandardCharsets.UTF_8);+ byte[] body = responseBody.get().getBytes(StandardCharsets.UTF_8);exchange.getResponseHeaders().set("Content-Type", "text/plain");exchange.sendResponseHeaders(status.get(), body.length);exchange.getResponseBody().write(body);@@ -158,6 +163,31 @@ class ApiGatewayTaskInvokerTest {verify(iam).unregisterSession(ACCOUNT, credentials.get()[0]);}+ @ParameterizedTest+ @ValueSource(strings = {"X-AmzFoo", "X-Amznn-Test", "X-ForwardedFoo", "X-Amz", "X-Amzn", "X-Forwarded"})+ void headersOutsideTheReservedPrefixesReachTheApi(String header) throws Exception {+ observedHeader.set(header);+ ((ObjectNode) request.get("Headers")).putArray(header).add("preserved");++ JsonNode result = invoker.invoke(request, machine, null);++ assertEquals(200, result.path("StatusCode").asInt());+ assertEquals(List.of("preserved"), receivedHeaders.get());+ assertEquals(null, serverFailure.get());+ verify(iam).unregisterSession(ACCOUNT, credentials.get()[0]);+ }++ @ParameterizedTest+ @CsvSource({"200 OK,200", "null trailing,null"})+ void scalarResponseUsesTheFirstJsonValueAsAwsDoes(String body, String expectedJson) throws Exception {+ responseBody.set(body);+ JsonNode result = invoker.invoke(request, machine, null);+ assertEquals(200, result.path("StatusCode").asInt());+ assertEquals(mapper.readTree(expectedJson), result.get("ResponseBody"));+ assertEquals(null, serverFailure.get());+ verify(iam).unregisterSession(ACCOUNT, credentials.get()[0]);+ }+@Testvoid httpErrorRemainsCatchableAndRevokesSession() {status.set(500);diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.javaindex 8288ecaad3..16e3755edd 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsValidateStateMachineDefinitionIntegrationTest.java@@ -13,6 +13,7 @@ import org.junit.jupiter.params.provider.Arguments;import org.junit.jupiter.params.provider.MethodSource;import java.util.List;+import java.util.Locale;import java.util.Map;import java.util.stream.Stream;@@ -156,6 +157,88 @@ class StepFunctionsValidateStateMachineDefinitionIntegrationTest {.when().post("/");}+ @ParameterizedTest+ @MethodSource("unsupportedApiHeaders")+ void apiGatewayReservedHeadersFailBeforeCreation(String header, boolean jsonata) {+ String definition = apiHeaderDefinition(header, jsonata);+ String field = jsonata ? "Arguments" : "Parameters";+ String message = "The 'Headers' field contains unsupported values: ["+ + header.toLowerCase(Locale.ROOT) + "]";+ validateDefinition(definition).then().statusCode(200)+ .body("result", equalTo("FAIL"))+ .body("diagnostics[0].code", equalTo("SCHEMA_VALIDATION_FAILED"))+ .body("diagnostics[0].message", equalTo(message))+ .body("diagnostics[0].location", equalTo("/States/Call/" + field));+ createStateMachine("invalid-header-" + System.nanoTime(), definition).then().statusCode(400)+ .body("__type", equalTo("InvalidDefinition"))+ .body("message", containsString(message));+ }++ static Stream<Arguments> unsupportedApiHeaders() {+ return Stream.of("Expect", "expect", "eXpEcT", "Upgrade", "upgrade", "UPGRADE",+ "Authorization", "authorization", "AUTHORIZATION", "Host", "host", "HOST",+ "X-Amz-Test", "x-amz-test", "X-AMZ-TEST", "X-Amzn-Test", "x-amzn-test", "X-AMZN-TEST",+ "X-Amz-", "X-Amzn-", "Connection", "Content-MD5", "Max-Forwards", "Proxy-Authenticate",+ "Server", "TE", "Transfer-Encoding", "Trailer", "Via", "WWW-Authenticate",+ "X-Forwarded-Test", "x-forwarded-test", "X-FORWARDED-TEST")+ .flatMap(header -> Stream.of(Arguments.of(header, false), Arguments.of(header, true)));+ }++ @ParameterizedTest+ @MethodSource("supportedApiHeaders")+ void apiGatewayOrdinaryHeadersRemainValid(String header, boolean jsonata) {+ validateDefinition(apiHeaderDefinition(header, jsonata)).then().statusCode(200)+ .body("result", equalTo("OK")).body("diagnostics", hasSize(0));+ }++ static Stream<Arguments> supportedApiHeaders() {+ return Stream.of("X_Custom", "X-Expect", "Content-Type", "X-Amz", "X-Amzn", "X-AmzFoo", "X-AmznFoo",+ "X-Amznn-Test", "Authorization-Extra", "Hostname", "Expect-Extra", "Upgrade-Extra",+ "X-Am-Example", "X-Am", "XX-Amz-Test", "X-Forwarded", "X-ForwardedFoo", "XX-Forwarded-Test")+ .flatMap(header -> Stream.of(Arguments.of(header, false), Arguments.of(header, true)));+ }++ @Test+ void apiGatewayHeaderValidationRecursesIntoParallelBranches() throws Exception {+ String definition = OBJECT_MAPPER.writeValueAsString(Map.of("StartAt", "P", "States", Map.of("P",+ Map.of("Type", "Parallel", "End", true,+ "Branches", List.of(OBJECT_MAPPER.readTree(apiHeaderDefinition("Expect", false)))))));+ validateDefinition(definition).then().statusCode(200).body("result", equalTo("FAIL"))+ .body("diagnostics[0].location", equalTo("/States/P/Branches[0]/States/Call/Parameters"));+ }++ @Test+ void invalidApiHeaderUpdatePreservesTheStoredDefinition() {+ String original = apiHeaderDefinition("X_Custom", false);+ String arn = createStateMachine("header-update-" + System.nanoTime(), original)+ .then().statusCode(200).extract().path("stateMachineArn");+ try {+ given().contentType(CT).header("X-Amz-Target", "AWSStepFunctions.UpdateStateMachine")+ .body(Map.of("stateMachineArn", arn, "definition", apiHeaderDefinition("Upgrade", false)))+ .post("/").then().statusCode(400).body("__type", equalTo("InvalidDefinition"));+ given().contentType(CT).header("X-Amz-Target", "AWSStepFunctions.DescribeStateMachine")+ .body(Map.of("stateMachineArn", arn)).post("/").then().statusCode(200)+ .body("definition", equalTo(original));+ } finally {+ given().contentType(CT).header("X-Amz-Target", "AWSStepFunctions.DeleteStateMachine")+ .body(Map.of("stateMachineArn", arn)).post("/").then().statusCode(200);+ }+ }++ private static String apiHeaderDefinition(String header, boolean jsonata) {+ ObjectNode root = OBJECT_MAPPER.createObjectNode().put("StartAt", "Call");+ if (jsonata) {+ root.put("QueryLanguage", "JSONata");+ }+ ObjectNode task = root.putObject("States").putObject("Call");+ task.put("Type", "Task").put("Resource", "arn:aws:states:::apigateway:invoke").put("End", true);+ ObjectNode input = task.putObject(jsonata ? "Arguments" : "Parameters");+ input.put("ApiEndpoint", "example.execute-api.us-east-2.amazonaws.com").put("Method", "POST")+ .put("AuthType", "NO_AUTH");+ input.putObject("Headers").putArray(header).add("test");+ return root.toString();+ }+private static String distributedMapWithResultWriter(String resultWriter) {return """{