Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(emr): scaling and termination policies and block public access (#4893)

David044c700d68b9
diff --git a/README.md b/README.md
index 02052703ae..9d852aa4ca 100644
--- a/README.md
+++ b/README.md
@@ -346,7 +346,7 @@ For operation-level compatibility, see the [Services Overview](https://floci.io/
| Amazon MQ | Real Docker | RabbitMQ broker via rabbitmq:3-management; AMQP + management console |
| Athena | In-process with DuckDB sidecar | Real SQL execution over S3 and Glue-backed views |
| Glue | In-process | Data Catalog, Schema Registry, tables consumed by Athena |
-| EMR | In-process | Cluster (job flow) lifecycle, instance groups and fleets, steps, security configurations, tagging |
+| EMR | In-process | Cluster (job flow) lifecycle, instance groups and fleets, steps, security configurations, scaling and termination policies, block public access, tagging |
| Data Firehose | In-process | Streaming delivery, buffered flush to S3 with GZIP/ZIP/Snappy compression |
| Managed Service for Apache Flink | Real Docker | Kinesis Analytics V2 control plane; StartApplication runs a real Flink cluster (JobManager + TaskManager, image per RuntimeEnvironment), pulls the application JAR from local S3, and submits the job |
| ECS | Real Docker | Clusters, task definitions, tasks, services, capacity providers, task sets |
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrTest.java
index e62a935e50..c2cb89f7d2 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrTest.java
@@ -160,6 +160,96 @@ class EmrTest {
@Test
@Order(11)
+ void managedScalingPolicyRoundTrip() {
+ emr.putManagedScalingPolicy(r -> r.clusterId(clusterId)
+ .managedScalingPolicy(ManagedScalingPolicy.builder()
+ .computeLimits(ComputeLimits.builder().unitType(ComputeLimitsUnitType.INSTANCES)
+ .minimumCapacityUnits(1).maximumCapacityUnits(5).build())
+ .build()));
+ ManagedScalingPolicy policy = emr.getManagedScalingPolicy(r -> r.clusterId(clusterId)).managedScalingPolicy();
+ assertThat(policy.computeLimits().unitType()).isEqualTo(ComputeLimitsUnitType.INSTANCES);
+ assertThat(policy.computeLimits().maximumCapacityUnits()).isEqualTo(5);
+
+ emr.removeManagedScalingPolicy(r -> r.clusterId(clusterId));
+ assertThat(emr.getManagedScalingPolicy(r -> r.clusterId(clusterId)).managedScalingPolicy()).isNull();
+ }
+
+ @Test
+ @Order(12)
+ void autoTerminationPolicyRoundTrip() {
+ emr.putAutoTerminationPolicy(r -> r.clusterId(clusterId)
+ .autoTerminationPolicy(AutoTerminationPolicy.builder().idleTimeout(3600L).build()));
+ assertThat(emr.getAutoTerminationPolicy(r -> r.clusterId(clusterId)).autoTerminationPolicy().idleTimeout())
+ .isEqualTo(3600L);
+
+ assertThatThrownBy(() -> emr.putAutoTerminationPolicy(r -> r.clusterId(clusterId)
+ .autoTerminationPolicy(AutoTerminationPolicy.builder().idleTimeout(30L).build())))
+ .isInstanceOf(InvalidRequestException.class);
+
+ emr.removeAutoTerminationPolicy(r -> r.clusterId(clusterId));
+ assertThat(emr.getAutoTerminationPolicy(r -> r.clusterId(clusterId)).autoTerminationPolicy()).isNull();
+ }
+
+ @Test
+ @Order(13)
+ void autoScalingPolicyOnCoreGroup() {
+ String coreGroupId = emr.listInstanceGroups(r -> r.clusterId(clusterId)).instanceGroups().stream()
+ .filter(g -> g.instanceGroupType() == InstanceGroupType.CORE)
+ .findFirst().orElseThrow().id();
+ PutAutoScalingPolicyResponse resp = emr.putAutoScalingPolicy(r -> r
+ .clusterId(clusterId)
+ .instanceGroupId(coreGroupId)
+ .autoScalingPolicy(AutoScalingPolicy.builder()
+ .constraints(ScalingConstraints.builder().minCapacity(1).maxCapacity(4).build())
+ .rules(ScalingRule.builder()
+ .name("scale-out")
+ .action(ScalingAction.builder()
+ .simpleScalingPolicyConfiguration(SimpleScalingPolicyConfiguration.builder()
+ .scalingAdjustment(1).build())
+ .build())
+ .trigger(ScalingTrigger.builder()
+ .cloudWatchAlarmDefinition(CloudWatchAlarmDefinition.builder()
+ .comparisonOperator(ComparisonOperator.LESS_THAN)
+ .metricName("YARNMemoryAvailablePercentage")
+ .period(300).threshold(15.0).build())
+ .build())
+ .build())
+ .build()));
+ assertThat(resp.instanceGroupId()).isEqualTo(coreGroupId);
+ assertThat(resp.autoScalingPolicy().status().state()).isEqualTo(AutoScalingPolicyState.ATTACHED);
+
+ InstanceGroup core = emr.listInstanceGroups(r -> r.clusterId(clusterId)).instanceGroups().stream()
+ .filter(g -> g.id().equals(coreGroupId)).findFirst().orElseThrow();
+ assertThat(core.autoScalingPolicy().constraints().maxCapacity()).isEqualTo(4);
+ assertThat(core.autoScalingPolicy().rules()).hasSize(1);
+
+ emr.removeAutoScalingPolicy(r -> r.clusterId(clusterId).instanceGroupId(coreGroupId));
+ InstanceGroup cleared = emr.listInstanceGroups(r -> r.clusterId(clusterId)).instanceGroups().stream()
+ .filter(g -> g.id().equals(coreGroupId)).findFirst().orElseThrow();
+ assertThat(cleared.autoScalingPolicy()).isNull();
+ }
+
+ @Test
+ @Order(14)
+ void blockPublicAccessConfigurationRoundTrip() {
+ GetBlockPublicAccessConfigurationResponse before = emr.getBlockPublicAccessConfiguration(r -> {});
+ assertThat(before.blockPublicAccessConfigurationMetadata().createdByArn()).isNotNull();
+
+ emr.putBlockPublicAccessConfiguration(r -> r.blockPublicAccessConfiguration(
+ BlockPublicAccessConfiguration.builder()
+ .blockPublicSecurityGroupRules(true)
+ .permittedPublicSecurityGroupRuleRanges(
+ PortRange.builder().minRange(22).maxRange(22).build(),
+ PortRange.builder().minRange(8443).build())
+ .build()));
+ BlockPublicAccessConfiguration after = emr.getBlockPublicAccessConfiguration(r -> {})
+ .blockPublicAccessConfiguration();
+ assertThat(after.blockPublicSecurityGroupRules()).isTrue();
+ assertThat(after.permittedPublicSecurityGroupRuleRanges()).hasSize(2);
+ }
+
+ @Test
+ @Order(20)
void terminate() {
emr.terminateJobFlows(r -> r.jobFlowIds(clusterId));
Cluster cluster = emr.describeCluster(r -> r.clusterId(clusterId)).cluster();
@@ -169,7 +259,7 @@ class EmrTest {
}
@Test
- @Order(12)
+ @Order(21)
void describeUnknownClusterThrowsInvalidRequest() {
assertThatThrownBy(() -> emr.describeCluster(r -> r.clusterId("j-DOESNOTEXIST0")))
.isInstanceOf(InvalidRequestException.class);
diff --git a/docs/services/emr.md b/docs/services/emr.md
index fac5ec4d7f..e1d571ebcc 100644
--- a/docs/services/emr.md
+++ b/docs/services/emr.md
@@ -35,6 +35,16 @@ Floci emulates the Amazon EMR (Elastic MapReduce) management API. Clusters (job
| `ListSecurityConfigurations` | Lists all security configurations |
| `AddTags` | Adds tags to a cluster |
| `RemoveTags` | Removes tags from a cluster |
+| `PutManagedScalingPolicy` | Sets a cluster's managed scaling policy |
+| `GetManagedScalingPolicy` | Returns a cluster's managed scaling policy |
+| `RemoveManagedScalingPolicy` | Removes a cluster's managed scaling policy |
+| `PutAutoTerminationPolicy` | Sets a cluster's idle timeout auto termination policy |
+| `GetAutoTerminationPolicy` | Returns a cluster's auto termination policy |
+| `RemoveAutoTerminationPolicy` | Removes a cluster's auto termination policy |
+| `PutAutoScalingPolicy` | Attaches an automatic scaling policy to a core or task instance group |
+| `RemoveAutoScalingPolicy` | Removes an instance group's automatic scaling policy |
+| `GetBlockPublicAccessConfiguration` | Returns the account's block public access configuration for the Region |
+| `PutBlockPublicAccessConfiguration` | Sets the account's block public access configuration for the Region |
<!-- floci:actions:end -->
## Example
diff --git a/docs/services/index.md b/docs/services/index.md
index 793122cc9d..d49be6ea62 100644
--- a/docs/services/index.md
+++ b/docs/services/index.md
@@ -77,7 +77,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac
| [Redshift](redshift.md) | `POST /` with `Action=` param + PostgreSQL container | Query + PostgreSQL wire (+ CFN) | 29 |
| [Redshift Data API](redshift-data.md) | `POST /` + `X-Amz-Target: RedshiftData.*` | JSON 1.1 | 11 |
| [Redshift Serverless](redshift-serverless.md) | `POST /` + `X-Amz-Target: RedshiftServerless.*` | JSON 1.1 | 8 |
-| [EMR](emr.md) | `POST /` + `X-Amz-Target: ElasticMapReduce.*` | JSON 1.1 | 24 |
+| [EMR](emr.md) | `POST /` + `X-Amz-Target: ElasticMapReduce.*` | JSON 1.1 | 34 |
| [EMR Serverless](emr-serverless.md) | `/applications/*`, `/tags/*` | REST JSON | 10 |
| [Data Firehose](firehose.md) | `POST /` + `X-Amz-Target: Firehose_20150804.*` | JSON 1.1 | 6 |
| [ECS](ecs.md) | `POST /` + `X-Amz-Target: AmazonEC2ContainerServiceV20141113.*` | JSON 1.1 | 58 |
diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java b/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java
index 29ca8b5d5b..e16af78632 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java
@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.common.AwsEndpoints;
import io.github.hectorvent.floci.core.common.AwsErrorResponse;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.AwsJson11Controller;
+import io.github.hectorvent.floci.services.emr.model.EmrBlockPublicAccess;
import io.github.hectorvent.floci.services.emr.model.EmrCluster;
import io.github.hectorvent.floci.services.emr.model.EmrInstanceFleet;
import io.github.hectorvent.floci.services.emr.model.EmrInstanceGroup;
@@ -70,6 +71,16 @@ public class EmrHandler {
case "ListSecurityConfigurations" -> handleListSecurityConfigurations();
case "AddTags" -> handleAddTags(request);
case "RemoveTags" -> handleRemoveTags(request);
+ case "PutManagedScalingPolicy" -> handlePutManagedScalingPolicy(request);
+ case "GetManagedScalingPolicy" -> handleGetManagedScalingPolicy(request);
+ case "RemoveManagedScalingPolicy" -> handleRemoveManagedScalingPolicy(request);
+ case "PutAutoTerminationPolicy" -> handlePutAutoTerminationPolicy(request);
+ case "GetAutoTerminationPolicy" -> handleGetAutoTerminationPolicy(request);
+ case "RemoveAutoTerminationPolicy" -> handleRemoveAutoTerminationPolicy(request);
+ case "PutAutoScalingPolicy" -> handlePutAutoScalingPolicy(request);
+ case "RemoveAutoScalingPolicy" -> handleRemoveAutoScalingPolicy(request);
+ case "GetBlockPublicAccessConfiguration" -> handleGetBlockPublicAccessConfiguration(region);
+ case "PutBlockPublicAccessConfiguration" -> handlePutBlockPublicAccessConfiguration(request, region);
default -> Response.status(400)
.entity(new AwsErrorResponse("InvalidRequestException",
"Operation " + action + " is not supported."))
@@ -106,6 +117,16 @@ public class EmrHandler {
cluster.setApplications(rawArray(request.path("Applications")));
cluster.setConfigurations(rawArray(request.path("Configurations")));
cluster.setTags(parseTags(request.path("Tags")));
+ JsonNode managedScaling = request.path("ManagedScalingPolicy");
+ if (!managedScaling.isMissingNode() && !managedScaling.isNull()) {
+ EmrService.validateManagedScalingPolicy(managedScaling);
+ cluster.setManagedScalingPolicy(managedScaling.toString());
+ }
+ JsonNode autoTermination = request.path("AutoTerminationPolicy");
+ if (!autoTermination.isMissingNode() && !autoTermination.isNull()) {
+ EmrService.validateAutoTerminationPolicy(autoTermination);
+ cluster.setAutoTerminationPolicy(autoTermination.toString());
+ }
JsonNode instances = request.path("Instances");
cluster.setKeepJobFlowAliveWhenNoSteps(instances.path("KeepJobFlowAliveWhenNoSteps").asBoolean(false));
@@ -325,6 +346,70 @@ public class EmrHandler {
// ──────────────────────────── Builders ────────────────────────────
+ private Response handlePutManagedScalingPolicy(JsonNode request) {
+ service.putManagedScalingPolicy(text(request, "ClusterId"), request.get("ManagedScalingPolicy"));
+ return Response.ok(objectMapper.createObjectNode()).build();
+ }
+
+ private Response handleGetManagedScalingPolicy(JsonNode request) {
+ ObjectNode response = objectMapper.createObjectNode();
+ setRawJson(response, "ManagedScalingPolicy", service.getManagedScalingPolicy(text(request, "ClusterId")));
+ return Response.ok(response).build();
+ }
+
+ private Response handleRemoveManagedScalingPolicy(JsonNode request) {
+ service.removeManagedScalingPolicy(text(request, "ClusterId"));
+ return Response.ok(objectMapper.createObjectNode()).build();
+ }
+
+ private Response handlePutAutoTerminationPolicy(JsonNode request) {
+ service.putAutoTerminationPolicy(text(request, "ClusterId"), request.path("AutoTerminationPolicy"));
+ return Response.ok(objectMapper.createObjectNode()).build();
+ }
+
+ private Response handleGetAutoTerminationPolicy(JsonNode request) {
+ ObjectNode response = objectMapper.createObjectNode();
+ setRawJson(response, "AutoTerminationPolicy", service.getAutoTerminationPolicy(text(request, "ClusterId")));
+ return Response.ok(response).build();
+ }
+
+ private Response handleRemoveAutoTerminationPolicy(JsonNode request) {
+ service.removeAutoTerminationPolicy(text(request, "ClusterId"));
+ return Response.ok(objectMapper.createObjectNode()).build();
+ }
+
+ private Response handlePutAutoScalingPolicy(JsonNode request) {
+ String clusterId = text(request, "ClusterId");
+ EmrInstanceGroup group = service.putAutoScalingPolicy(clusterId, text(request, "InstanceGroupId"),
+ request.get("AutoScalingPolicy"));
+ ObjectNode response = objectMapper.createObjectNode();
+ response.put("ClusterId", clusterId);
+ response.put("InstanceGroupId", group.getId());
+ response.set("AutoScalingPolicy", autoScalingPolicyNode(group.getAutoScalingPolicy()));
+ response.put("ClusterArn", service.describeCluster(clusterId).getClusterArn());
+ return Response.ok(response).build();
+ }
+
+ private Response handleRemoveAutoScalingPolicy(JsonNode request) {
+ service.removeAutoScalingPolicy(text(request, "ClusterId"), text(request, "InstanceGroupId"));
+ return Response.ok(objectMapper.createObjectNode()).build();
+ }
+
+ private Response handleGetBlockPublicAccessConfiguration(String region) {
+ EmrBlockPublicAccess setting = service.getBlockPublicAccess(region);
+ ObjectNode response = objectMapper.createObjectNode();
+ setRawJson(response, "BlockPublicAccessConfiguration", setting.getConfiguration());
+ ObjectNode metadata = response.putObject("BlockPublicAccessConfigurationMetadata");
+ putEpoch(metadata, "CreationDateTime", setting.getCreationDateTime());
+ metadata.put("CreatedByArn", setting.getCreatedByArn());
+ return Response.ok(response).build();
+ }
+
+ private Response handlePutBlockPublicAccessConfiguration(JsonNode request, String region) {
+ service.putBlockPublicAccess(region, request.get("BlockPublicAccessConfiguration"));
+ return Response.ok(objectMapper.createObjectNode()).build();
+ }
+
private ObjectNode clusterNode(EmrCluster c) {
ObjectNode node = objectMapper.createObjectNode();
node.put("Id", c.getId());
@@ -441,6 +526,23 @@ public class EmrHandler {
node.put("RequestedInstanceCount", g.getRequestedInstanceCount());
node.put("RunningInstanceCount", g.getRunningInstanceCount());
node.putObject("Status").put("State", g.getState());
+ if (g.getAutoScalingPolicy() != null) {
+ node.set("AutoScalingPolicy", autoScalingPolicyNode(g.getAutoScalingPolicy()));
+ }
+ return node;
+ }
+
+ /** An AutoScalingPolicyDescription: the stored Constraints and Rules with an ATTACHED status. */
+ private ObjectNode autoScalingPolicyNode(String rawPolicy) {
+ ObjectNode node = objectMapper.createObjectNode();
+ node.putObject("Status").put("State", "ATTACHED");
+ try {
+ JsonNode policy = objectMapper.readTree(rawPolicy);
+ node.set("Constraints", policy.path("Constraints"));
+ node.set("Rules", policy.path("Rules"));
+ } catch (Exception e) {
+ LOG.warnv("Failed to parse stored AutoScalingPolicy JSON; policy omitted. Value: {0}", rawPolicy);
+ }
return node;
}
@@ -526,6 +628,11 @@ public class EmrHandler {
group.setMarket(g.path("Market").asText(null));
group.setBidPrice(g.path("BidPrice").asText(null));
group.setRequestedInstanceCount(g.path("InstanceCount").asInt(0));
+ JsonNode autoScaling = g.path("AutoScalingPolicy");
+ if (!autoScaling.isMissingNode() && !autoScaling.isNull()) {
+ EmrService.validateAutoScalingPolicy(autoScaling, group.getInstanceGroupType());
+ group.setAutoScalingPolicy(autoScaling.toString());
+ }
groups.add(group);
}
return groups;
diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java b/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java
index 44c953ca92..689b4d833f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java
@@ -1,11 +1,13 @@
package io.github.hectorvent.floci.services.emr;
import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.emr.model.EmrBlockPublicAccess;
import io.github.hectorvent.floci.services.emr.model.EmrCluster;
import io.github.hectorvent.floci.services.emr.model.EmrInstanceFleet;
import io.github.hectorvent.floci.services.emr.model.EmrInstanceGroup;
@@ -19,6 +21,7 @@ import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* EMR management-plane business logic. Cluster and step lifecycles are simulated
@@ -34,7 +37,19 @@ public class EmrService {
private final StorageBackend<String, EmrCluster> clusterStore;
private final StorageBackend<String, SecurityConfiguration> secConfigStore;
+ // Block public access is an account setting per Region, so this store is keyed by Region.
+ private final StorageBackend<String, EmrBlockPublicAccess> blockPublicAccessStore;
private final RegionResolver regionResolver;
+
+ private static final Set<String> COMPUTE_LIMITS_UNIT_TYPES = Set.of("InstanceFleetUnits", "Instances", "VCPU");
+ private static final Set<String> SCALING_STRATEGIES = Set.of("DEFAULT", "ADVANCED");
+ private static final long MIN_IDLE_TIMEOUT_SECONDS = 60;
+ private static final long MAX_IDLE_TIMEOUT_SECONDS = 604_800;
+ private static final int MAX_PORT = 65_535;
+ // The AWS default: block public access enabled in every Region, with port 22 as an exception.
+ private static final String DEFAULT_BLOCK_PUBLIC_ACCESS =
+ "{\"BlockPublicSecurityGroupRules\":true,"
+ + "\"PermittedPublicSecurityGroupRuleRanges\":[{\"MinRange\":22,\"MaxRange\":22}]}";
private final String defaultReleaseLabel;
@Inject
@@ -43,13 +58,15 @@ public class EmrService {
new TypeReference<Map<String, EmrCluster>>() {});
this.secConfigStore = storageFactory.create("emr", "emr-security-configs.json",
new TypeReference<Map<String, SecurityConfiguration>>() {});
+ this.blockPublicAccessStore = storageFactory.create("emr", "emr-block-public-access.json",
+ new TypeReference<Map<String, EmrBlockPublicAccess>>() {});
this.regionResolver = regionResolver;
this.defaultReleaseLabel = config.services().emr().defaultReleaseLabel();
}
// ──────────────────────────── Cluster lifecycle ────────────────────────────
- public EmrCluster runJobFlow(EmrCluster cluster, String region) {
+ public synchronized EmrCluster runJobFlow(EmrCluster cluster, String region) {
// Only a missing Name fails AWS's framework validation: the member is REQUIRED but its
// constraint is len=[0..256], so an explicit empty string is accepted by real EMR.
if (cluster.getName() == null) {
@@ -107,7 +124,7 @@ public class EmrService {
return result;
}
- public void terminateJobFlows(List<String> ids) {
+ public synchronized void terminateJobFlows(List<String> ids) {
// AWS terminates the unprotected clusters in the request and then fails it with a
// ValidationException if any were termination protected.
boolean anyProtected = false;
@@ -135,26 +152,26 @@ public class EmrService {
}
}
- public void setTerminationProtection(List<String> ids, boolean value) {
+ public synchronized void setTerminationProtection(List<String> ids, boolean value) {
mutateClusters(ids, c -> c.setTerminationProtected(value));
}
- public void setVisibleToAllUsers(List<String> ids, boolean value) {
+ public synchronized void setVisibleToAllUsers(List<String> ids, boolean value) {
mutateClusters(ids, c -> c.setVisibleToAllUsers(value));
}
- public void setKeepJobFlowAliveWhenNoSteps(List<String> ids, boolean value) {
+ public synchronized void setKeepJobFlowAliveWhenNoSteps(List<String> ids, boolean value) {
mutateClusters(ids, c -> {
c.setKeepJobFlowAliveWhenNoSteps(value);
c.setAutoTerminate(!value);
});
}
- public void setUnhealthyNodeReplacement(List<String> ids, boolean value) {
+ public synchronized void setUnhealthyNodeReplacement(List<String> ids, boolean value) {
mutateClusters(ids, c -> c.setUnhealthyNodeReplacement(value));
}
- public int modifyCluster(String id, Integer stepConcurrencyLevel) {
+ public synchronized int modifyCluster(String id, Integer stepConcurrencyLevel) {
EmrCluster cluster = requireCluster(id);
if (stepConcurrencyLevel != null) {
cluster.setStepConcurrencyLevel(stepConcurrencyLevel);
@@ -165,7 +182,7 @@ public class EmrService {
// ──────────────────────────── Steps ────────────────────────────
- public List<String> addJobFlowSteps(String clusterId, List<EmrStep> steps) {
+ public synchronized List<String> addJobFlowSteps(String clusterId, List<EmrStep> steps) {
EmrCluster cluster = clusterStore.get(clusterId).orElseThrow(() -> new AwsException(
"InvalidRequestException", "Cluster id '" + clusterId + "' is not valid.", 400));
List<String> ids = new ArrayList<>();
@@ -208,7 +225,7 @@ public class EmrService {
public record CancelStepInfo(String stepId, String status, String reason) {}
- public List<CancelStepInfo> cancelSteps(String clusterId, List<String> stepIds) {
+ public synchronized List<CancelStepInfo> cancelSteps(String clusterId, List<String> stepIds) {
EmrCluster cluster = requireCluster(clusterId);
List<CancelStepInfo> result = new ArrayList<>();
for (String stepId : stepIds) {
@@ -231,7 +248,7 @@ public class EmrService {
// ──────────────────────────── Instance groups / fleets ────────────────────────────
- public List<String> addInstanceGroups(String clusterId, List<EmrInstanceGroup> groups) {
+ public synchronized List<String> addInstanceGroups(String clusterId, List<EmrInstanceGroup> groups) {
EmrCluster cluster = requireCluster(clusterId);
List<String> ids = new ArrayList<>();
for (EmrInstanceGroup group : groups) {
@@ -249,7 +266,7 @@ public class EmrService {
return requireCluster(clusterId).getInstanceGroups();
}
- public String addInstanceFleet(String clusterId, EmrInstanceFleet fleet) {
+ public synchronized String addInstanceFleet(String clusterId, EmrInstanceFleet fleet) {
EmrCluster cluster = requireCluster(clusterId);
fleet.setId("if-" + randomId(13));
fleet.setProvisionedOnDemandCapacity(fleet.getTargetOnDemandCapacity());
@@ -305,13 +322,13 @@ public class EmrService {
// ──────────────────────────── Tags ────────────────────────────
- public void addTags(String resourceId, Map<String, String> tags) {
+ public synchronized void addTags(String resourceId, Map<String, String> tags) {
EmrCluster cluster = requireCluster(resourceId);
cluster.getTags().putAll(tags);
clusterStore.put(resourceId, cluster);
}
- public void removeTags(String resourceId, List<String> keys) {
+ public synchronized void removeTags(String resourceId, List<String> keys) {
EmrCluster cluster = requireCluster(resourceId);
keys.forEach(cluster.getTags()::remove);
clusterStore.put(resourceId, cluster);
@@ -319,6 +336,211 @@ public class EmrService {
// ──────────────────────────── Helpers ────────────────────────────
+ // ──────────────────────────── Scaling and termination policies ────────────────────────────
+
+ public synchronized void putManagedScalingPolicy(String clusterId, JsonNode policy) {
+ EmrCluster cluster = requireCluster(clusterId);
+ validateManagedScalingPolicy(policy);
+ cluster.setManagedScalingPolicy(policy.toString());
+ clusterStore.put(clusterId, cluster);
+ }
+
+ /** The cluster's managed scaling policy as raw JSON, or null when it has none. */
+ public String getManagedScalingPolicy(String clusterId) {
+ return requireCluster(clusterId).getManagedScalingPolicy();
+ }
+
+ public synchronized void removeManagedScalingPolicy(String clusterId) {
+ EmrCluster cluster = requireCluster(clusterId);
+ cluster.setManagedScalingPolicy(null);
+ clusterStore.put(clusterId, cluster);
+ }
+
+ /** AutoTerminationPolicy is optional in the request; an absent one is stored as an empty policy. */
+ public synchronized void putAutoTerminationPolicy(String clusterId, JsonNode policy) {
+ EmrCluster cluster = requireCluster(clusterId);
+ validateAutoTerminationPolicy(policy);
+ cluster.setAutoTerminationPolicy(policy == null || policy.isMissingNode() || policy.isNull() ? "{}" : policy.toString());
+ clusterStore.put(clusterId, cluster);
+ }
+
+ /** The cluster's auto termination policy as raw JSON, or null when it has none. */
+ public String getAutoTerminationPolicy(String clusterId) {
+ return requireCluster(clusterId).getAutoTerminationPolicy();
+ }
+
+ public synchronized void removeAutoTerminationPolicy(String clusterId) {
+ EmrCluster cluster = requireCluster(clusterId);
+ cluster.setAutoTerminationPolicy(null);
+ clusterStore.put(clusterId, cluster);
+ }
+
+ /** Attaches an automatic scaling policy to a core or task instance group and returns the group. */
+ public synchronized EmrInstanceGroup putAutoScalingPolicy(String clusterId, String instanceGroupId, JsonNode policy) {
+ EmrCluster cluster = requireCluster(clusterId);
+ EmrInstanceGroup group = requireInstanceGroup(cluster, instanceGroupId);
+ validateAutoScalingPolicy(policy, group.getInstanceGroupType());
+ group.setAutoScalingPolicy(policy.toString());
+ clusterStore.put(clusterId, cluster);
+ return group;
+ }
+
+ public synchronized void removeAutoScalingPolicy(String clusterId, String instanceGroupId) {
+ EmrCluster cluster = requireCluster(clusterId);
+ requireInstanceGroup(cluster, instanceGroupId).setAutoScalingPolicy(null);
+ clusterStore.put(clusterId, cluster);
+ }
+
+ // ──────────────────────────── Block public access ────────────────────────────
+
+ /**
+ * The account's block public access configuration in {@code region}. Until one is put, it is the
+ * AWS default: enabled for every Region, with port 22 as an exception. The default is stored the
+ * first time it is read, so its metadata stays the same on later reads.
+ */
+ public synchronized EmrBlockPublicAccess getBlockPublicAccess(String region) {
+ return blockPublicAccessStore.get(region).orElseGet(() -> {
+ EmrBlockPublicAccess setting = new EmrBlockPublicAccess();
+ setting.setConfiguration(DEFAULT_BLOCK_PUBLIC_ACCESS);
+ setting.setCreationDateTime(Instant.now());
+ setting.setCreatedByArn(regionResolver.buildGlobalArn("iam", "root"));
+ blockPublicAccessStore.put(region, setting);
+ return setting;
+ });
+ }
+
+ public synchronized void putBlockPublicAccess(String region, JsonNode configuration) {
+ validateBlockPublicAccess(configuration);
+ EmrBlockPublicAccess setting = new EmrBlockPublicAccess();
+ setting.setConfiguration(configuration.toString());
+ setting.setCreationDateTime(Instant.now());
+ setting.setCreatedByArn(regionResolver.buildGlobalArn("iam", "root"));
+ blockPublicAccessStore.put(region, setting);
+ }
+
+ // ──────────────────────────── Policy validation (from the API model) ────────────────────────────
+
+ /** ComputeLimits is required, with UnitType, MinimumCapacityUnits and MaximumCapacityUnits. */
+ static void validateManagedScalingPolicy(JsonNode policy) {
+ if (policy == null || !policy.isObject()) {
+ throw invalid("ManagedScalingPolicy is required.");
+ }
+ JsonNode limits = policy.path("ComputeLimits");
+ if (!limits.isObject()) {
+ throw invalid("ManagedScalingPolicy.ComputeLimits is required.");
+ }
+ if (!COMPUTE_LIMITS_UNIT_TYPES.contains(limits.path("UnitType").asText(""))) {
+ throw invalid("ComputeLimits.UnitType must be one of " + COMPUTE_LIMITS_UNIT_TYPES + ".");
+ }
+ if (!limits.path("MinimumCapacityUnits").isInt() || !limits.path("MaximumCapacityUnits").isInt()) {
+ throw invalid("ComputeLimits.MinimumCapacityUnits and MaximumCapacityUnits are required.");
+ }
+ int min = limits.get("MinimumCapacityUnits").asInt();
+ int max = limits.get("MaximumCapacityUnits").asInt();
+ if (min < 0 || min > max) {
+ throw invalid("ComputeLimits.MinimumCapacityUnits must be between 0 and MaximumCapacityUnits.");
+ }
+ for (String optional : List.of("MaximumOnDemandCapacityUnits", "MaximumCoreCapacityUnits")) {
+ JsonNode value = limits.path(optional);
+ if (!value.isMissingNode() && (!value.isInt() || value.asInt() < 0 || value.asInt() > max)) {
+ throw invalid("ComputeLimits." + optional + " must be between 0 and MaximumCapacityUnits.");
+ }
+ }
+ JsonNode strategy = policy.path("ScalingStrategy");
+ if (!strategy.isMissingNode() && !SCALING_STRATEGIES.contains(strategy.asText(""))) {
+ throw invalid("ScalingStrategy must be one of " + SCALING_STRATEGIES + ".");
+ }
+ }
+
+ /** IdleTimeout, when given, is 60 to 604800 seconds. */
+ static void validateAutoTerminationPolicy(JsonNode policy) {
+ if (policy == null || policy.isMissingNode() || policy.isNull()) {
+ return;
+ }
+ if (!policy.isObject()) {
+ throw invalid("AutoTerminationPolicy must be an object.");
+ }
+ JsonNode idle = policy.path("IdleTimeout");
+ if (!idle.isMissingNode() && (!idle.canConvertToLong()
+ || idle.asLong() < MIN_IDLE_TIMEOUT_SECONDS || idle.asLong() > MAX_IDLE_TIMEOUT_SECONDS)) {
+ throw invalid("AutoTerminationPolicy.IdleTimeout must be between " + MIN_IDLE_TIMEOUT_SECONDS
+ + " and " + MAX_IDLE_TIMEOUT_SECONDS + " seconds.");
+ }
+ }
+
+ /**
+ * Constraints (MinCapacity not above MaxCapacity) and at least one rule with Name, Action and
+ * Trigger are required; only core and task instance groups scale automatically.
+ */
+ static void validateAutoScalingPolicy(JsonNode policy, String instanceGroupType) {
+ if (policy == null || !policy.isObject()) {
+ throw invalid("AutoScalingPolicy is required.");
+ }
+ if (!"CORE".equals(instanceGroupType) && !"TASK".equals(instanceGroupType)) {
+ throw invalid("An automatic scaling policy can only be attached to a core or task instance group.");
+ }
+ JsonNode constraints = policy.path("Constraints");
+ if (!constraints.path("MinCapacity").isInt() || !constraints.path("MaxCapacity").isInt()) {
+ throw invalid("AutoScalingPolicy.Constraints.MinCapacity and MaxCapacity are required.");
+ }
+ if (constraints.get("MinCapacity").asInt() < 0
+ || constraints.get("MinCapacity").asInt() > constraints.get("MaxCapacity").asInt()) {
+ throw invalid("Constraints.MinCapacity must be between 0 and MaxCapacity.");
+ }
+ JsonNode rules = policy.path("Rules");
+ if (!rules.isArray() || rules.isEmpty()) {
+ throw invalid("AutoScalingPolicy.Rules must contain at least one rule.");
+ }
+ for (JsonNode rule : rules) {
+ if (rule.path("Name").asText("").isEmpty() || !rule.path("Action").isObject() || !rule.path("Trigger").isObject()) {
+ throw invalid("Each scaling rule needs a Name, an Action and a Trigger.");
+ }
+ }
+ }
+
+ /** BlockPublicSecurityGroupRules is required; each port range needs MinRange, within 0 to 65535. */
+ static void validateBlockPublicAccess(JsonNode configuration) {
+ if (configuration == null || !configuration.isObject()
+ || !configuration.path("BlockPublicSecurityGroupRules").isBoolean()) {
+ throw invalid("BlockPublicAccessConfiguration.BlockPublicSecurityGroupRules is required.");
+ }
+ JsonNode ranges = configuration.path("PermittedPublicSecurityGroupRuleRanges");
+ if (ranges.isMissingNode()) {
+ return;
+ }
+ if (!ranges.isArray()) {
+ throw invalid("PermittedPublicSecurityGroupRuleRanges must be a list.");
+ }
+ for (JsonNode range : ranges) {
+ if (!range.path("MinRange").isInt()) {
+ throw invalid("Each port range needs a MinRange.");
+ }
+ int min = range.get("MinRange").asInt();
+ JsonNode maxRange = range.path("MaxRange");
+ if (!maxRange.isMissingNode() && !maxRange.isInt()) {
+ throw invalid("Port range MaxRange must be an integer.");
+ }
+ int max = maxRange.isInt() ? maxRange.asInt() : min;
+ if (min < 0 || max > MAX_PORT || min > max) {
+ throw invalid("Port range " + min + "-" + max + " must lie within 0-" + MAX_PORT + ", MinRange first.");
+ }
+ }
+ }
+
+ private static AwsException invalid(String message) {
+ return new AwsException("InvalidRequestException", message, 400);
+ }
+
+ private static EmrInstanceGroup requireInstanceGroup(EmrCluster cluster, String instanceGroupId) {
+ if (instanceGroupId == null) {
+ throw invalid("InstanceGroupId is required.");
+ }
+ return cluster.getInstanceGroups().stream()
+ .filter(group -> instanceGroupId.equals(group.getId()))
+ .findFirst()
+ .orElseThrow(() -> invalid("Instance group " + instanceGroupId + " is not in cluster " + cluster.getId() + "."));
+ }
+
private EmrCluster requireCluster(String id) {
if (id == null) {
throw new AwsException("InvalidRequestException", "ClusterId is required.", 400);
diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/model/EmrBlockPublicAccess.java b/src/main/java/io/github/hectorvent/floci/services/emr/model/EmrBlockPublicAccess.java
new file mode 100644
index 0000000000..13e950ffe9
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/emr/model/EmrBlockPublicAccess.java
@@ -0,0 +1,26 @@
+package io.github.hectorvent.floci.services.emr.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+
+/** The block public access configuration of an account in one Region, and when and by whom it was set. */
+@RegisterForReflection
+public class EmrBlockPublicAccess {
+
+ /** The BlockPublicAccessConfiguration as given, raw JSON. */
+ private String configuration;
+ private Instant creationDateTime;
+ private String createdByArn;
+
+ public EmrBlockPublicAccess() {}
+
+ public String getConfiguration() { return configuration; }
+ public void setConfiguration(String configuration) { this.configuration = configuration; }
+
+ public Instant getCreationDateTime() { return creationDateTime; }
+ public void setCreationDateTime(Instant creationDateTime) { this.creationDateTime = creationDateTime; }
+
+ public String getCreatedByArn() { return createdByArn; }
+ public void setCreatedByArn(String createdByArn) { this.createdByArn = createdByArn; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/model/EmrCluster.java b/src/main/java/io/github/hectorvent/floci/services/emr/model/EmrCluster.java
index aa63255f33..d686ffe45f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emr/model/EmrCluster.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emr/model/EmrCluster.java
@@ -44,6 +44,9 @@ public class EmrCluster {
private String ec2Attributes;
private String applications;
private String configurations;
+ // The cluster's ManagedScalingPolicy and AutoTerminationPolicy as given, raw JSON; null when none.
+ private String managedScalingPolicy;
+ private String autoTerminationPolicy;
private String region;
private Instant creationDateTime;
private Instant readyDateTime;
@@ -154,6 +157,12 @@ public class EmrCluster {
public String getConfigurations() { return configurations; }
public void setConfigurations(String configurations) { this.configurations = configurations; }
+ public String getManagedScalingPolicy() { return managedScalingPolicy; }
+ public void setManagedScalingPolicy(String managedScalingPolicy) { this.managedScalingPolicy = managedScalingPolicy; }
+
+ public String getAutoTerminationPolicy() { return autoTerminationPolicy; }
+ public void setAutoTerminationPolicy(String autoTerminationPolicy) { this.autoTerminationPolicy = autoTerminationPolicy; }
+
public String getRegion() { return region; }
public void setRegion(String region) { this.region = region; }
diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/model/EmrInstanceGroup.java b/src/main/java/io/github/hectorvent/floci/services/emr/model/EmrInstanceGroup.java
index 8f469e679f..92f7ee2419 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emr/model/EmrInstanceGroup.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emr/model/EmrInstanceGroup.java
@@ -15,6 +15,8 @@ public class EmrInstanceGroup {
private int requestedInstanceCount;
private int runningInstanceCount;
private String state;
+ /** The AutoScalingPolicy as given (Constraints and Rules), raw JSON; null when none is attached. */
+ private String autoScalingPolicy;
public EmrInstanceGroup() {}
@@ -48,4 +50,7 @@ public class EmrInstanceGroup {
public String getState() { return state; }
public void setState(String state) { this.state = state; }
+
+ public String getAutoScalingPolicy() { return autoScalingPolicy; }
+ public void setAutoScalingPolicy(String autoScalingPolicy) { this.autoScalingPolicy = autoScalingPolicy; }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emr/EmrPoliciesIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emr/EmrPoliciesIntegrationTest.java
new file mode 100644
index 0000000000..6ba3d92793
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/emr/EmrPoliciesIntegrationTest.java
@@ -0,0 +1,190 @@
+package io.github.hectorvent.floci.services.emr;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.anEmptyMap;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ * EMR managed scaling, auto termination and automatic scaling policies, and the account's block
+ * public access configuration, over the AWS JSON 1.1 wire protocol.
+ */
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class EmrPoliciesIntegrationTest {
+
+ private static final String CT = "application/x-amz-json-1.1";
+ private static final String PREFIX = "ElasticMapReduce.";
+ private static final String MANAGED_SCALING =
+ "{\"ComputeLimits\":{\"UnitType\":\"Instances\",\"MinimumCapacityUnits\":1,\"MaximumCapacityUnits\":5}}";
+ private static final String AUTO_SCALING =
+ "{\"Constraints\":{\"MinCapacity\":1,\"MaxCapacity\":4},\"Rules\":[{\"Name\":\"scale-out\","
+ + "\"Action\":{\"SimpleScalingPolicyConfiguration\":{\"ScalingAdjustment\":1}},"
+ + "\"Trigger\":{\"CloudWatchAlarmDefinition\":{\"ComparisonOperator\":\"LESS_THAN\","
+ + "\"MetricName\":\"YARNMemoryAvailablePercentage\",\"Period\":300,\"Threshold\":15}}}]}";
+
+ private static String clusterId;
+ private static String masterGroupId;
+ private static String coreGroupId;
+
+ @BeforeAll
+ static void configure() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ private static Response call(String action, String body) {
+ return given().contentType(CT).header("X-Amz-Target", PREFIX + action)
+ .body(body).when().post("/");
+ }
+
+ private static String cluster() {
+ return "{\"ClusterId\":\"" + clusterId + "\"";
+ }
+
+ @Test
+ @Order(1)
+ void runJobFlowStoresPolicies() {
+ Response resp = call("RunJobFlow",
+ "{\"Name\":\"policies\",\"ReleaseLabel\":\"emr-7.5.0\","
+ + "\"ManagedScalingPolicy\":" + MANAGED_SCALING + ","
+ + "\"AutoTerminationPolicy\":{\"IdleTimeout\":3600},"
+ + "\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true,"
+ + "\"InstanceGroups\":[{\"InstanceRole\":\"MASTER\",\"InstanceType\":\"m5.xlarge\",\"InstanceCount\":1},"
+ + "{\"InstanceRole\":\"CORE\",\"InstanceType\":\"m5.xlarge\",\"InstanceCount\":2,"
+ + "\"AutoScalingPolicy\":" + AUTO_SCALING + "}]}}");
+ resp.then().statusCode(200);
+ clusterId = resp.jsonPath().getString("JobFlowId");
+
+ call("GetManagedScalingPolicy", cluster() + "}").then().statusCode(200)
+ .body("ManagedScalingPolicy.ComputeLimits.MaximumCapacityUnits", equalTo(5));
+ call("GetAutoTerminationPolicy", cluster() + "}").then().statusCode(200)
+ .body("AutoTerminationPolicy.IdleTimeout", equalTo(3600));
+ Response groups = call("ListInstanceGroups", cluster() + "}");
+ groups.then().statusCode(200)
+ .body("InstanceGroups.find { it.InstanceGroupType == 'CORE' }.AutoScalingPolicy.Status.State",
+ equalTo("ATTACHED"))
+ .body("InstanceGroups.find { it.InstanceGroupType == 'CORE' }.AutoScalingPolicy.Constraints.MaxCapacity",
+ equalTo(4))
+ .body("InstanceGroups.find { it.InstanceGroupType == 'MASTER' }.AutoScalingPolicy", nullValue());
+ masterGroupId = groups.jsonPath().getString("InstanceGroups.find { it.InstanceGroupType == 'MASTER' }.Id");
+ coreGroupId = groups.jsonPath().getString("InstanceGroups.find { it.InstanceGroupType == 'CORE' }.Id");
+ }
+
+ @Test
+ @Order(2)
+ void managedScalingPolicyPutGetRemove() {
+ call("PutManagedScalingPolicy", cluster() + ",\"ManagedScalingPolicy\":{\"ComputeLimits\":"
+ + "{\"UnitType\":\"VCPU\",\"MinimumCapacityUnits\":4,\"MaximumCapacityUnits\":32},"
+ + "\"ScalingStrategy\":\"ADVANCED\"}}").then().statusCode(200);
+ call("GetManagedScalingPolicy", cluster() + "}").then().statusCode(200)
+ .body("ManagedScalingPolicy.ComputeLimits.UnitType", equalTo("VCPU"))
+ .body("ManagedScalingPolicy.ScalingStrategy", equalTo("ADVANCED"));
+ call("RemoveManagedScalingPolicy", cluster() + "}").then().statusCode(200);
+ call("GetManagedScalingPolicy", cluster() + "}").then().statusCode(200)
+ .body("$", anEmptyMap());
+ }
+
+ @Test
+ @Order(3)
+ void managedScalingPolicyIsValidated() {
+ call("PutManagedScalingPolicy", cluster() + ",\"ManagedScalingPolicy\":{}}")
+ .then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
+ call("PutManagedScalingPolicy", cluster() + ",\"ManagedScalingPolicy\":{\"ComputeLimits\":"
+ + "{\"UnitType\":\"Instances\",\"MinimumCapacityUnits\":6,\"MaximumCapacityUnits\":5}}}")
+ .then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
+ call("PutManagedScalingPolicy", cluster() + ",\"ManagedScalingPolicy\":{\"ComputeLimits\":"
+ + "{\"UnitType\":\"Instances\",\"MinimumCapacityUnits\":1,\"MaximumCapacityUnits\":5,"
+ + "\"MaximumOnDemandCapacityUnits\":-1}}}")
+ .then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
+ call("PutManagedScalingPolicy", "{\"ClusterId\":\"j-MISSING\",\"ManagedScalingPolicy\":" + MANAGED_SCALING + "}")
+ .then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
+ }
+
+ @Test
+ @Order(4)
+ void autoTerminationPolicyPutGetRemove() {
+ call("PutAutoTerminationPolicy", cluster() + ",\"AutoTerminationPolicy\":{\"IdleTimeout\":120}}")
+ .then().statusCode(200);
+ call("GetAutoTerminationPolicy", cluster() + "}").then().statusCode(200)
+ .body("AutoTerminationPolicy.IdleTimeout", equalTo(120));
+ call("PutAutoTerminationPolicy", cluster() + ",\"AutoTerminationPolicy\":{\"IdleTimeout\":59}}")
+ .then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
+ call("PutAutoTerminationPolicy", cluster() + ",\"AutoTerminationPolicy\":{\"IdleTimeout\":604801}}")
+ .then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
+ call("RemoveAutoTerminationPolicy", cluster() + "}").then().statusCode(200);
+ call("GetAutoTerminationPolicy", cluster() + "}").then().statusCode(200)
+ .body("$", anEmptyMap());
+ }
+
+ @Test
+ @Order(5)
+ void autoScalingPolicyPutAndRemove() {
+ call("PutAutoScalingPolicy", cluster() + ",\"InstanceGroupId\":\"" + coreGroupId + "\","
+ + "\"AutoScalingPolicy\":{\"Constraints\":{\"MinCapacity\":2,\"MaxCapacity\":8},\"Rules\":"
+ + "[{\"Name\":\"r\",\"Action\":{\"SimpleScalingPolicyConfiguration\":{\"ScalingAdjustment\":2}},"
+ + "\"Trigger\":{\"CloudWatchAlarmDefinition\":{\"ComparisonOperator\":\"GREATER_THAN\","
+ + "\"MetricName\":\"m\",\"Period\":300,\"Threshold\":80}}}]}}")
+ .then().statusCode(200)
+ .body("ClusterId", equalTo(clusterId))
+ .body("InstanceGroupId", equalTo(coreGroupId))
+ .body("ClusterArn", notNullValue())
+ .body("AutoScalingPolicy.Status.State", equalTo("ATTACHED"))
+ .body("AutoScalingPolicy.Constraints.MaxCapacity", equalTo(8))
+ .body("AutoScalingPolicy.Rules", hasSize(1));
+ call("RemoveAutoScalingPolicy", cluster() + ",\"InstanceGroupId\":\"" + coreGroupId + "\"}")
+ .then().statusCode(200);
+ call("ListInstanceGroups", cluster() + "}").then().statusCode(200)
+ .body("InstanceGroups.find { it.Id == '" + coreGroupId + "' }.AutoScalingPolicy", nullValue());
+ }
+
+ @Test
+ @Order(6)
+ void autoScalingPolicyOnlyOnCoreOrTaskGroups() {
+ call("PutAutoScalingPolicy", cluster() + ",\"InstanceGroupId\":\"" + masterGroupId + "\","
+ + "\"AutoScalingPolicy\":" + AUTO_SCALING + "}")
+ .then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
+ call("PutAutoScalingPolicy", cluster() + ",\"InstanceGroupId\":\"ig-MISSING\","
+ + "\"AutoScalingPolicy\":" + AUTO_SCALING + "}")
+ .then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
+ call("RunJobFlow", "{\"Name\":\"bad\",\"Instances\":{\"InstanceGroups\":[{\"InstanceRole\":\"MASTER\","
+ + "\"InstanceType\":\"m5.xlarge\",\"InstanceCount\":1,\"AutoScalingPolicy\":" + AUTO_SCALING + "}]}}")
+ .then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
+ }
+
+ @Test
+ @Order(7)
+ void blockPublicAccessDefaultsThenPut() {
+ call("GetBlockPublicAccessConfiguration", "{}").then().statusCode(200)
+ .body("BlockPublicAccessConfiguration.BlockPublicSecurityGroupRules", equalTo(true))
+ .body("BlockPublicAccessConfiguration.PermittedPublicSecurityGroupRuleRanges[0].MinRange", equalTo(22))
+ .body("BlockPublicAccessConfigurationMetadata.CreatedByArn", notNullValue())
+ .body("BlockPublicAccessConfigurationMetadata.CreationDateTime", notNullValue());
+ call("PutBlockPublicAccessConfiguration", "{\"BlockPublicAccessConfiguration\":"
+ + "{\"BlockPublicSecurityGroupRules\":true,\"PermittedPublicSecurityGroupRuleRanges\":"
+ + "[{\"MinRange\":22,\"MaxRange\":22},{\"MinRange\":100,\"MaxRange\":101}]}}")
+ .then().statusCode(200);
+ call("GetBlockPublicAccessConfiguration", "{}").then().statusCode(200)
+ .body("BlockPublicAccessConfiguration.PermittedPublicSecurityGroupRuleRanges", hasSize(2));
+ call("PutBlockPublicAccessConfiguration", "{\"BlockPublicAccessConfiguration\":"
+ + "{\"BlockPublicSecurityGroupRules\":true,\"PermittedPublicSecurityGroupRuleRanges\":"
+ + "[{\"MinRange\":70000}]}}")
+ .then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
+ call("PutBlockPublicAccessConfiguration", "{\"BlockPublicAccessConfiguration\":"
+ + "{\"BlockPublicSecurityGroupRules\":true,\"PermittedPublicSecurityGroupRuleRanges\":"
+ + "[{\"MinRange\":22,\"MaxRange\":\"invalid\"}]}}")
+ .then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
+ call("PutBlockPublicAccessConfiguration", "{\"BlockPublicAccessConfiguration\":{}}")
+ .then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
+ }
+}