Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(emr): terminate transient clusters after steps (#6085)

Jimmy Liu Jia67e4bfec1c22
diff --git a/docs/services/emr.md b/docs/services/emr.md
index c982f36c1c..618de87df0 100644
--- a/docs/services/emr.md
+++ b/docs/services/emr.md
@@ -53,6 +53,15 @@ Floci emulates the Amazon EMR (Elastic MapReduce) management API. Clusters (job
| `PutBlockPublicAccessConfiguration` | Sets the account's block public access configuration for the Region |
<!-- floci:actions:end -->
+## Cluster lifecycle
+
+Steps complete immediately in this metadata-only emulator. A cluster created with
+`Instances.KeepJobFlowAliveWhenNoSteps` set to `false` terminates after its initial steps or
+steps added later complete, reporting `ALL_STEPS_COMPLETED`. A cluster with no steps remains
+`WAITING` until steps are added. Set `KeepJobFlowAliveWhenNoSteps` to `true` to keep the cluster
+`WAITING` after its steps complete. Automatic step-completion termination also applies when
+termination protection is enabled.
+
## Release catalogue
`ListReleaseLabels`, `DescribeReleaseLabel` and `ListSupportedInstanceTypes` read a curated catalogue,
@@ -80,7 +89,7 @@ export AWS_ENDPOINT_URL=http://localhost:4566
CLUSTER_ID=$(aws emr run-job-flow \
--name "demo-cluster" \
--release-label emr-7.0.0 \
- --instances InstanceGroups='[{InstanceCount=1,InstanceGroupType=MASTER,InstanceType=m5.xlarge}]' \
+ --instances KeepJobFlowAliveWhenNoSteps=true,InstanceGroups='[{InstanceCount=1,InstanceGroupType=MASTER,InstanceType=m5.xlarge}]' \
--query 'JobFlowId' --output text)
# Inspect it
diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java b/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java
index fb55cd4e53..4297caf32b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java
@@ -34,8 +34,9 @@ import java.util.function.Function;
/**
* EMR management-plane business logic. Cluster and step lifecycles are simulated
- * through immediate state transitions (no Hadoop runs): clusters land in WAITING and
- * initial steps in COMPLETED. The {@code clusterStartupDelaySeconds} config exists for
+ * through immediate state transitions (no Hadoop runs): steps complete immediately,
+ * leaving persistent clusters WAITING and transient clusters with steps TERMINATED.
+ * The {@code clusterStartupDelaySeconds} config exists for
* future deferred transitions; Phase 1 advances synchronously for deterministic tests.
*/
@ApplicationScoped
@@ -111,6 +112,7 @@ public class EmrService {
completeStep(step);
}
advanceToWaiting(cluster);
+ terminateAfterCompletedSteps(cluster);
clusterStore.put(id, cluster);
return cluster;
}
@@ -147,11 +149,11 @@ public class EmrService {
if (cluster == null) {
continue;
}
- if (cluster.isTerminationProtected()) {
- anyProtected = true;
+ if ("TERMINATED".equals(cluster.getState())) {
continue;
}
- if ("TERMINATED".equals(cluster.getState())) {
+ if (cluster.isTerminationProtected()) {
+ anyProtected = true;
continue;
}
cluster.setState("TERMINATED");
@@ -205,6 +207,9 @@ public class EmrService {
cluster.getSteps().add(step);
ids.add(step.getId());
}
+ if (!steps.isEmpty()) {
+ terminateAfterCompletedSteps(cluster);
+ }
clusterStore.put(clusterId, cluster);
return ids;
}
@@ -780,6 +785,17 @@ public class EmrService {
cluster.setReadyDateTime(Instant.now());
}
+ private void terminateAfterCompletedSteps(EmrCluster cluster) {
+ if (cluster.getSteps().isEmpty() || !cluster.isAutoTerminate()
+ || "TERMINATED".equals(cluster.getState())) {
+ return;
+ }
+ cluster.setState("TERMINATED");
+ cluster.setStateChangeReasonCode("ALL_STEPS_COMPLETED");
+ cluster.setStateChangeMessage("All steps completed.");
+ cluster.setEndDateTime(Instant.now());
+ }
+
private void initStep(EmrStep step) {
step.setId("s-" + randomId(13));
step.setState("PENDING");
diff --git a/src/test/java/io/github/hectorvent/floci/services/emr/EmrTransientIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emr/EmrTransientIntegrationTest.java
new file mode 100644
index 0000000000..4e0cd0296d
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/emr/EmrTransientIntegrationTest.java
@@ -0,0 +1,131 @@
+package io.github.hectorvent.floci.services.emr;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import static io.github.hectorvent.floci.testing.EmrRequests.emr;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+@QuarkusTest
+class EmrTransientIntegrationTest {
+
+ @BeforeAll
+ static void configure() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void runJobFlowWithStepsTerminatesDespiteTerminationProtection() {
+ String clusterId = emr("RunJobFlow",
+ "{\"Name\":\"transient-initial\",\"Instances\":{\"TerminationProtected\":true},"
+ + "\"Steps\":[{\"Name\":\"initial\","
+ + "\"HadoopJarStep\":{\"Jar\":\"command-runner.jar\"}}]}")
+ .then().statusCode(200).extract().path("JobFlowId");
+
+ emr("DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
+ .then().statusCode(200)
+ .body("Cluster.AutoTerminate", equalTo(true))
+ .body("Cluster.TerminationProtected", equalTo(true))
+ .body("Cluster.Status.State", equalTo("TERMINATED"))
+ .body("Cluster.Status.StateChangeReason.Code", equalTo("ALL_STEPS_COMPLETED"))
+ .body("Cluster.Status.Timeline.EndDateTime", notNullValue());
+ }
+
+ @Test
+ void addJobFlowStepsTerminatesTransientCluster() {
+ String clusterId = emr("RunJobFlow", "{\"Name\":\"transient-later\",\"Instances\":{}}")
+ .then().statusCode(200).extract().path("JobFlowId");
+ emr("DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
+ .then().statusCode(200)
+ .body("Cluster.Status.State", equalTo("WAITING"));
+
+ emr("AddJobFlowSteps",
+ "{\"JobFlowId\":\"" + clusterId + "\",\"Steps\":[{\"Name\":\"later\","
+ + "\"HadoopJarStep\":{\"Jar\":\"command-runner.jar\"}}]}")
+ .then().statusCode(200);
+ emr("DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
+ .then().statusCode(200)
+ .body("Cluster.Status.State", equalTo("TERMINATED"))
+ .body("Cluster.Status.StateChangeReason.Code", equalTo("ALL_STEPS_COMPLETED"))
+ .body("Cluster.Status.Timeline.EndDateTime", notNullValue());
+ }
+
+ @Test
+ void keepAliveClusterWaitsAfterItsSteps() {
+ String clusterId = emr("RunJobFlow",
+ "{\"Name\":\"keep-alive\",\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true},"
+ + "\"Steps\":[{\"Name\":\"initial\","
+ + "\"HadoopJarStep\":{\"Jar\":\"command-runner.jar\"}}]}")
+ .then().statusCode(200).extract().path("JobFlowId");
+
+ emr("DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
+ .then().statusCode(200)
+ .body("Cluster.AutoTerminate", equalTo(false))
+ .body("Cluster.Status.State", equalTo("WAITING"))
+ .body("Cluster.Status.Timeline.ReadyDateTime", notNullValue())
+ .body("Cluster.Status.Timeline.EndDateTime", nullValue());
+ }
+
+ @Test
+ void transientClusterWithoutStepsRemainsWaiting() {
+ String clusterId = emr("RunJobFlow", "{\"Name\":\"no-steps\",\"Instances\":{}}")
+ .then().statusCode(200).extract().path("JobFlowId");
+
+ emr("DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
+ .then().statusCode(200)
+ .body("Cluster.AutoTerminate", equalTo(true))
+ .body("Cluster.Status.State", equalTo("WAITING"))
+ .body("Cluster.Status.Timeline.EndDateTime", nullValue());
+ }
+
+ @Test
+ void addingStepsAfterUserTerminationPreservesReasonAndEndTime() {
+ String clusterId = emr("RunJobFlow", "{\"Name\":\"manually-ended-transient\",\"Instances\":{}}")
+ .then().statusCode(200).extract().path("JobFlowId");
+ String clusterRequest = "{\"ClusterId\":\"" + clusterId + "\"}";
+ emr("TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}")
+ .then().statusCode(200);
+ Object endTime = emr("DescribeCluster", clusterRequest).then().statusCode(200)
+ .body("Cluster.Status.StateChangeReason.Code", equalTo("USER_REQUEST"))
+ .extract().path("Cluster.Status.Timeline.EndDateTime");
+
+ emr("AddJobFlowSteps",
+ "{\"JobFlowId\":\"" + clusterId + "\",\"Steps\":[{\"Name\":\"late\","
+ + "\"HadoopJarStep\":{\"Jar\":\"command-runner.jar\"}}]}")
+ .then().statusCode(200);
+ emr("ListSteps", clusterRequest).then().statusCode(200).body("Steps", hasSize(1));
+ emr("DescribeCluster", clusterRequest).then().statusCode(200)
+ .body("Cluster.Status.State", equalTo("TERMINATED"))
+ .body("Cluster.Status.StateChangeReason.Code", equalTo("USER_REQUEST"))
+ .body("Cluster.Status.Timeline.EndDateTime", equalTo(endTime));
+ }
+
+ @Test
+ void protectedCompletedClusterIsSkippedDuringMixedTermination() {
+ String completedId = emr("RunJobFlow",
+ "{\"Name\":\"protected-completed\",\"Instances\":{\"TerminationProtected\":true},"
+ + "\"Steps\":[{\"Name\":\"initial\",\"HadoopJarStep\":{\"Jar\":\"command-runner.jar\"}}]}")
+ .then().statusCode(200).extract().path("JobFlowId");
+ String activeId = emr("RunJobFlow",
+ "{\"Name\":\"active\",\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true}}")
+ .then().statusCode(200).extract().path("JobFlowId");
+ String completedRequest = "{\"ClusterId\":\"" + completedId + "\"}";
+ Object endTime = emr("DescribeCluster", completedRequest).then().statusCode(200)
+ .extract().path("Cluster.Status.Timeline.EndDateTime");
+
+ emr("TerminateJobFlows", "{\"JobFlowIds\":[\"" + completedId + "\",\"" + activeId + "\"]}")
+ .then().statusCode(200);
+ emr("DescribeCluster", completedRequest).then().statusCode(200)
+ .body("Cluster.Status.State", equalTo("TERMINATED"))
+ .body("Cluster.Status.StateChangeReason.Code", equalTo("ALL_STEPS_COMPLETED"))
+ .body("Cluster.Status.Timeline.EndDateTime", equalTo(endTime));
+ emr("DescribeCluster", "{\"ClusterId\":\"" + activeId + "\"}").then().statusCode(200)
+ .body("Cluster.Status.State", equalTo("TERMINATED"))
+ .body("Cluster.Status.StateChangeReason.Code", equalTo("USER_REQUEST"));
+ }
+}