Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(emr): preserve request-level step execution role (#6061)

Jimmy Liu Jiaae53c4b35236
diff --git a/docs/services/emr.md b/docs/services/emr.md
index 9598ba7e71..c982f36c1c 100644
--- a/docs/services/emr.md
+++ b/docs/services/emr.md
@@ -94,3 +94,7 @@ aws emr add-steps --cluster-id "$CLUSTER_ID" \
# Terminate
aws emr terminate-clusters --cluster-ids "$CLUSTER_ID"
```
+
+`aws emr add-steps --execution-role-arn` applies the role to every step in that request.
+`RunJobFlow` also applies `StepExecutionRoleArn` to its initial steps. `describe-step`
+includes `ExecutionRoleArn` for those steps; `list-steps` returns step summaries without it.
diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java b/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java
index f4f10f554a..beb3ad0122 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java
@@ -148,7 +148,7 @@ public class EmrHandler {
cluster.setEc2Attributes(buildEc2Attributes(instances));
cluster.setInstanceGroups(parseInstanceGroups(instances.path("InstanceGroups")));
cluster.setInstanceFleets(parseInstanceFleets(instances.path("InstanceFleets")));
- cluster.setSteps(parseSteps(request.path("Steps")));
+ cluster.setSteps(parseSteps(request.path("Steps"), text(request, "StepExecutionRoleArn")));
EmrCluster created = service.runJobFlow(cluster, region);
ObjectNode response = objectMapper.createObjectNode();
@@ -212,7 +212,8 @@ public class EmrHandler {
// ──────────────────────────── Steps ────────────────────────────
private Response handleAddJobFlowSteps(JsonNode request) {
- List<String> ids = service.addJobFlowSteps(text(request, "JobFlowId"), parseSteps(request.path("Steps")));
+ List<String> ids = service.addJobFlowSteps(text(request, "JobFlowId"),
+ parseSteps(request.path("Steps"), text(request, "ExecutionRoleArn")));
ObjectNode response = objectMapper.createObjectNode();
ArrayNode arr = response.putArray("StepIds");
ids.forEach(arr::add);
@@ -222,7 +223,7 @@ public class EmrHandler {
private Response handleDescribeStep(JsonNode request) {
EmrStep step = service.describeStep(text(request, "ClusterId"), text(request, "StepId"));
ObjectNode response = objectMapper.createObjectNode();
- response.set("Step", stepNode(step));
+ response.set("Step", stepNode(step, true));
return Response.ok(response).build();
}
@@ -232,7 +233,7 @@ public class EmrHandler {
ObjectNode response = objectMapper.createObjectNode();
ArrayNode arr = response.putArray("Steps");
for (EmrStep step : steps) {
- arr.add(stepNode(step));
+ arr.add(stepNode(step, false));
}
return Response.ok(response).build();
}
@@ -606,7 +607,7 @@ public class EmrHandler {
return status;
}
- private ObjectNode stepNode(EmrStep step) {
+ private ObjectNode stepNode(EmrStep step, boolean includeExecutionRole) {
ObjectNode node = objectMapper.createObjectNode();
node.put("Id", step.getId());
node.put("Name", step.getName());
@@ -622,6 +623,9 @@ public class EmrHandler {
ObjectNode props = config.putObject("Properties");
step.getProperties().forEach(props::put);
node.put("ActionOnFailure", step.getActionOnFailure());
+ if (includeExecutionRole && step.getExecutionRoleArn() != null) {
+ node.put("ExecutionRoleArn", step.getExecutionRoleArn());
+ }
ObjectNode status = node.putObject("Status");
status.put("State", step.getState());
if (step.getStateChangeReason() != null) {
@@ -723,7 +727,7 @@ public class EmrHandler {
// ──────────────────────────── Parsing ────────────────────────────
- private List<EmrStep> parseSteps(JsonNode stepsNode) {
+ private List<EmrStep> parseSteps(JsonNode stepsNode, String executionRoleArn) {
List<EmrStep> steps = new ArrayList<>();
if (!stepsNode.isArray()) {
return steps;
@@ -732,7 +736,7 @@ public class EmrHandler {
EmrStep step = new EmrStep();
step.setName(s.path("Name").asText(null));
step.setActionOnFailure(s.path("ActionOnFailure").asText(null));
- step.setExecutionRoleArn(s.path("ExecutionRoleArn").asText(null));
+ step.setExecutionRoleArn(executionRoleArn);
JsonNode jar = s.path("HadoopJarStep");
step.setJar(jar.path("Jar").asText(null));
step.setMainClass(jar.path("MainClass").asText(null));
diff --git a/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java
index 7e7401b04b..4941d5490f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java
@@ -13,7 +13,9 @@ import org.junit.jupiter.api.TestMethodOrder;
import static io.github.hectorvent.floci.testing.EmrRequests.emr;
import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.startsWith;
@@ -40,8 +42,10 @@ class EmrIntegrationTest {
@Test
@Order(1)
void runJobFlow() {
+ String roleArn = "arn:aws:iam::000000000000:role/emr-initial-step";
Response resp = emr("RunJobFlow",
"{\"Name\":\"floci-emr\",\"ReleaseLabel\":\"emr-7.5.0\","
+ + "\"StepExecutionRoleArn\":\"" + roleArn + "\","
+ "\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true,"
+ "\"InstanceGroups\":[{\"Name\":\"master\",\"InstanceRole\":\"MASTER\","
+ "\"InstanceType\":\"m5.xlarge\",\"InstanceCount\":1},"
@@ -53,6 +57,11 @@ class EmrIntegrationTest {
.body("JobFlowId", startsWith("j-"))
.body("ClusterArn", notNullValue());
clusterId = resp.jsonPath().getString("JobFlowId");
+ String initialStepId = emr("ListSteps", "{\"ClusterId\":\"" + clusterId + "\"}")
+ .then().statusCode(200).extract().jsonPath().getString("Steps[0].Id");
+ emr("DescribeStep", "{\"ClusterId\":\"" + clusterId + "\",\"StepId\":\"" + initialStepId + "\"}")
+ .then().statusCode(200)
+ .body("Step.ExecutionRoleArn", equalTo(roleArn));
}
@Test
@@ -132,6 +141,30 @@ class EmrIntegrationTest {
@Test
@Order(9)
+ void addStepsPreservesRequestLevelExecutionRole() {
+ String roleArn = "arn:aws:iam::000000000000:role/emr-step";
+ Response added = emr("AddJobFlowSteps",
+ "{\"JobFlowId\":\"" + clusterId + "\",\"ExecutionRoleArn\":\"" + roleArn + "\","
+ + "\"Steps\":[{\"Name\":\"first-role-step\",\"HadoopJarStep\":{\"Jar\":\"command-runner.jar\"}},"
+ + "{\"Name\":\"second-role-step\",\"HadoopJarStep\":{\"Jar\":\"command-runner.jar\"}}]}");
+ added.then().statusCode(200).body("StepIds", hasSize(2));
+ String firstId = added.jsonPath().getString("StepIds[0]");
+ String secondId = added.jsonPath().getString("StepIds[1]");
+
+ emr("DescribeStep", "{\"ClusterId\":\"" + clusterId + "\",\"StepId\":\"" + firstId + "\"}")
+ .then().statusCode(200)
+ .body("Step.ExecutionRoleArn", equalTo(roleArn));
+ emr("DescribeStep", "{\"ClusterId\":\"" + clusterId + "\",\"StepId\":\"" + secondId + "\"}")
+ .then().statusCode(200)
+ .body("Step.ExecutionRoleArn", equalTo(roleArn));
+ emr("ListSteps", "{\"ClusterId\":\"" + clusterId + "\"}")
+ .then().statusCode(200)
+ .body("Steps.find { it.Id == '" + firstId + "' }", not(hasKey("ExecutionRoleArn")))
+ .body("Steps.find { it.Id == '" + secondId + "' }", not(hasKey("ExecutionRoleArn")));
+ }
+
+ @Test
+ @Order(10)
void modifyStepConcurrency() {
emr("ModifyCluster", "{\"ClusterId\":\"" + clusterId + "\",\"StepConcurrencyLevel\":5}")
.then().statusCode(200)
@@ -139,7 +172,7 @@ class EmrIntegrationTest {
}
@Test
- @Order(10)
+ @Order(11)
void securityConfigurationRoundTrip() {
emr("CreateSecurityConfiguration",
"{\"Name\":\"floci-emr-sec\",\"SecurityConfiguration\":\"{\\\"EncryptionConfiguration\\\":{}}\"}")
@@ -155,7 +188,7 @@ class EmrIntegrationTest {
}
@Test
- @Order(11)
+ @Order(12)
void tagsRoundTrip() {
emr("AddTags", "{\"ResourceId\":\"" + clusterId + "\",\"Tags\":[{\"Key\":\"env\",\"Value\":\"test\"}]}")
.then().statusCode(200);
@@ -167,7 +200,7 @@ class EmrIntegrationTest {
}
@Test
- @Order(12)
+ @Order(13)
void terminateCluster() {
emr("TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}").then().statusCode(200);
emr("DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
@@ -177,7 +210,7 @@ class EmrIntegrationTest {
}
@Test
- @Order(13)
+ @Order(14)
void describeUnknownClusterIsInvalidRequest() {
emr("DescribeCluster", "{\"ClusterId\":\"j-DOESNOTEXIST0\"}")
.then().statusCode(400)
@@ -185,7 +218,7 @@ class EmrIntegrationTest {
}
@Test
- @Order(14)
+ @Order(15)
void runJobFlowWithoutNameIsValidationException() {
emr("RunJobFlow", "{\"ReleaseLabel\":\"emr-7.5.0\",\"Instances\":{}}")
.then().statusCode(400)
@@ -196,7 +229,7 @@ class EmrIntegrationTest {
}
@Test
- @Order(15)
+ @Order(16)
void runJobFlowWithEmptyNameIsAccepted() {
// The Name member is REQUIRED but its model constraint is len=[0..256], so real EMR
// accepts an explicit empty string; only a missing Name fails framework validation.
@@ -206,7 +239,7 @@ class EmrIntegrationTest {
}
@Test
- @Order(16)
+ @Order(17)
void addStepsToUnknownClusterIsInvalidRequest() {
emr("AddJobFlowSteps",
"{\"JobFlowId\":\"j-DOESNOTEXIST0\",\"Steps\":[{\"Name\":\"s\","
@@ -217,7 +250,7 @@ class EmrIntegrationTest {
}
@Test
- @Order(17)
+ @Order(18)
void terminationProtectionIsValidationExceptionAndUnprotectedStillTerminate() {
String protectedId = emr("RunJobFlow",
"{\"Name\":\"protected\",\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true}}")