Light, fluffy, and always free - The AWS Local Emulator alternative
fix(emr): preserve request-level step execution role (#6061)
Jimmy Liu Jiaae53c4b35236
diff --git a/docs/services/emr.md b/docs/services/emr.mdindex 9598ba7e71..c982f36c1c 100644--- a/docs/services/emr.md+++ b/docs/services/emr.md@@ -94,3 +94,7 @@ aws emr add-steps --cluster-id "$CLUSTER_ID" \# Terminateaws emr terminate-clusters --cluster-ids "$CLUSTER_ID"```++`aws emr add-steps --execution-role-arn` applies the role to every step in that request.+`RunJobFlow` also applies `StepExecutionRoleArn` to its initial steps. `describe-step`+includes `ExecutionRoleArn` for those steps; `list-steps` returns step summaries without it.diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java b/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.javaindex f4f10f554a..beb3ad0122 100644--- a/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java@@ -148,7 +148,7 @@ public class EmrHandler {cluster.setEc2Attributes(buildEc2Attributes(instances));cluster.setInstanceGroups(parseInstanceGroups(instances.path("InstanceGroups")));cluster.setInstanceFleets(parseInstanceFleets(instances.path("InstanceFleets")));- cluster.setSteps(parseSteps(request.path("Steps")));+ cluster.setSteps(parseSteps(request.path("Steps"), text(request, "StepExecutionRoleArn")));EmrCluster created = service.runJobFlow(cluster, region);ObjectNode response = objectMapper.createObjectNode();@@ -212,7 +212,8 @@ public class EmrHandler {// ──────────────────────────── Steps ────────────────────────────private Response handleAddJobFlowSteps(JsonNode request) {- List<String> ids = service.addJobFlowSteps(text(request, "JobFlowId"), parseSteps(request.path("Steps")));+ List<String> ids = service.addJobFlowSteps(text(request, "JobFlowId"),+ parseSteps(request.path("Steps"), text(request, "ExecutionRoleArn")));ObjectNode response = objectMapper.createObjectNode();ArrayNode arr = response.putArray("StepIds");ids.forEach(arr::add);@@ -222,7 +223,7 @@ public class EmrHandler {private Response handleDescribeStep(JsonNode request) {EmrStep step = service.describeStep(text(request, "ClusterId"), text(request, "StepId"));ObjectNode response = objectMapper.createObjectNode();- response.set("Step", stepNode(step));+ response.set("Step", stepNode(step, true));return Response.ok(response).build();}@@ -232,7 +233,7 @@ public class EmrHandler {ObjectNode response = objectMapper.createObjectNode();ArrayNode arr = response.putArray("Steps");for (EmrStep step : steps) {- arr.add(stepNode(step));+ arr.add(stepNode(step, false));}return Response.ok(response).build();}@@ -606,7 +607,7 @@ public class EmrHandler {return status;}- private ObjectNode stepNode(EmrStep step) {+ private ObjectNode stepNode(EmrStep step, boolean includeExecutionRole) {ObjectNode node = objectMapper.createObjectNode();node.put("Id", step.getId());node.put("Name", step.getName());@@ -622,6 +623,9 @@ public class EmrHandler {ObjectNode props = config.putObject("Properties");step.getProperties().forEach(props::put);node.put("ActionOnFailure", step.getActionOnFailure());+ if (includeExecutionRole && step.getExecutionRoleArn() != null) {+ node.put("ExecutionRoleArn", step.getExecutionRoleArn());+ }ObjectNode status = node.putObject("Status");status.put("State", step.getState());if (step.getStateChangeReason() != null) {@@ -723,7 +727,7 @@ public class EmrHandler {// ──────────────────────────── Parsing ────────────────────────────- private List<EmrStep> parseSteps(JsonNode stepsNode) {+ private List<EmrStep> parseSteps(JsonNode stepsNode, String executionRoleArn) {List<EmrStep> steps = new ArrayList<>();if (!stepsNode.isArray()) {return steps;@@ -732,7 +736,7 @@ public class EmrHandler {EmrStep step = new EmrStep();step.setName(s.path("Name").asText(null));step.setActionOnFailure(s.path("ActionOnFailure").asText(null));- step.setExecutionRoleArn(s.path("ExecutionRoleArn").asText(null));+ step.setExecutionRoleArn(executionRoleArn);JsonNode jar = s.path("HadoopJarStep");step.setJar(jar.path("Jar").asText(null));step.setMainClass(jar.path("MainClass").asText(null));diff --git a/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.javaindex 7e7401b04b..4941d5490f 100644--- a/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java@@ -13,7 +13,9 @@ import org.junit.jupiter.api.TestMethodOrder;import static io.github.hectorvent.floci.testing.EmrRequests.emr;import static org.hamcrest.Matchers.endsWith;import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.hasKey;import static org.hamcrest.Matchers.hasSize;+import static org.hamcrest.Matchers.not;import static org.hamcrest.Matchers.notNullValue;import static org.hamcrest.Matchers.startsWith;@@ -40,8 +42,10 @@ class EmrIntegrationTest {@Test@Order(1)void runJobFlow() {+ String roleArn = "arn:aws:iam::000000000000:role/emr-initial-step";Response resp = emr("RunJobFlow","{\"Name\":\"floci-emr\",\"ReleaseLabel\":\"emr-7.5.0\","+ + "\"StepExecutionRoleArn\":\"" + roleArn + "\","+ "\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true,"+ "\"InstanceGroups\":[{\"Name\":\"master\",\"InstanceRole\":\"MASTER\","+ "\"InstanceType\":\"m5.xlarge\",\"InstanceCount\":1},"@@ -53,6 +57,11 @@ class EmrIntegrationTest {.body("JobFlowId", startsWith("j-")).body("ClusterArn", notNullValue());clusterId = resp.jsonPath().getString("JobFlowId");+ String initialStepId = emr("ListSteps", "{\"ClusterId\":\"" + clusterId + "\"}")+ .then().statusCode(200).extract().jsonPath().getString("Steps[0].Id");+ emr("DescribeStep", "{\"ClusterId\":\"" + clusterId + "\",\"StepId\":\"" + initialStepId + "\"}")+ .then().statusCode(200)+ .body("Step.ExecutionRoleArn", equalTo(roleArn));}@Test@@ -132,6 +141,30 @@ class EmrIntegrationTest {@Test@Order(9)+ void addStepsPreservesRequestLevelExecutionRole() {+ String roleArn = "arn:aws:iam::000000000000:role/emr-step";+ Response added = emr("AddJobFlowSteps",+ "{\"JobFlowId\":\"" + clusterId + "\",\"ExecutionRoleArn\":\"" + roleArn + "\","+ + "\"Steps\":[{\"Name\":\"first-role-step\",\"HadoopJarStep\":{\"Jar\":\"command-runner.jar\"}},"+ + "{\"Name\":\"second-role-step\",\"HadoopJarStep\":{\"Jar\":\"command-runner.jar\"}}]}");+ added.then().statusCode(200).body("StepIds", hasSize(2));+ String firstId = added.jsonPath().getString("StepIds[0]");+ String secondId = added.jsonPath().getString("StepIds[1]");++ emr("DescribeStep", "{\"ClusterId\":\"" + clusterId + "\",\"StepId\":\"" + firstId + "\"}")+ .then().statusCode(200)+ .body("Step.ExecutionRoleArn", equalTo(roleArn));+ emr("DescribeStep", "{\"ClusterId\":\"" + clusterId + "\",\"StepId\":\"" + secondId + "\"}")+ .then().statusCode(200)+ .body("Step.ExecutionRoleArn", equalTo(roleArn));+ emr("ListSteps", "{\"ClusterId\":\"" + clusterId + "\"}")+ .then().statusCode(200)+ .body("Steps.find { it.Id == '" + firstId + "' }", not(hasKey("ExecutionRoleArn")))+ .body("Steps.find { it.Id == '" + secondId + "' }", not(hasKey("ExecutionRoleArn")));+ }++ @Test+ @Order(10)void modifyStepConcurrency() {emr("ModifyCluster", "{\"ClusterId\":\"" + clusterId + "\",\"StepConcurrencyLevel\":5}").then().statusCode(200)@@ -139,7 +172,7 @@ class EmrIntegrationTest {}@Test- @Order(10)+ @Order(11)void securityConfigurationRoundTrip() {emr("CreateSecurityConfiguration","{\"Name\":\"floci-emr-sec\",\"SecurityConfiguration\":\"{\\\"EncryptionConfiguration\\\":{}}\"}")@@ -155,7 +188,7 @@ class EmrIntegrationTest {}@Test- @Order(11)+ @Order(12)void tagsRoundTrip() {emr("AddTags", "{\"ResourceId\":\"" + clusterId + "\",\"Tags\":[{\"Key\":\"env\",\"Value\":\"test\"}]}").then().statusCode(200);@@ -167,7 +200,7 @@ class EmrIntegrationTest {}@Test- @Order(12)+ @Order(13)void terminateCluster() {emr("TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}").then().statusCode(200);emr("DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")@@ -177,7 +210,7 @@ class EmrIntegrationTest {}@Test- @Order(13)+ @Order(14)void describeUnknownClusterIsInvalidRequest() {emr("DescribeCluster", "{\"ClusterId\":\"j-DOESNOTEXIST0\"}").then().statusCode(400)@@ -185,7 +218,7 @@ class EmrIntegrationTest {}@Test- @Order(14)+ @Order(15)void runJobFlowWithoutNameIsValidationException() {emr("RunJobFlow", "{\"ReleaseLabel\":\"emr-7.5.0\",\"Instances\":{}}").then().statusCode(400)@@ -196,7 +229,7 @@ class EmrIntegrationTest {}@Test- @Order(15)+ @Order(16)void runJobFlowWithEmptyNameIsAccepted() {// The Name member is REQUIRED but its model constraint is len=[0..256], so real EMR// accepts an explicit empty string; only a missing Name fails framework validation.@@ -206,7 +239,7 @@ class EmrIntegrationTest {}@Test- @Order(16)+ @Order(17)void addStepsToUnknownClusterIsInvalidRequest() {emr("AddJobFlowSteps","{\"JobFlowId\":\"j-DOESNOTEXIST0\",\"Steps\":[{\"Name\":\"s\","@@ -217,7 +250,7 @@ class EmrIntegrationTest {}@Test- @Order(17)+ @Order(18)void terminationProtectionIsValidationExceptionAndUnprotectedStillTerminate() {String protectedId = emr("RunJobFlow","{\"Name\":\"protected\",\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true}}")