Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(emr): ListReleaseLabels, DescribeReleaseLabel and ListSupportedInstanceTypes (#5748)

David7dc514b52f2c
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrTest.java
index d40aeb3845..a15e1ecf08 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EmrTest.java
@@ -323,4 +323,26 @@ class EmrTest {
assertThatThrownBy(() -> emr.describeCluster(r -> r.clusterId("j-DOESNOTEXIST0")))
.isInstanceOf(InvalidRequestException.class);
}
+
+ @Test
+ @Order(22)
+ void releaseCatalogue() {
+ assertThat(emr.listReleaseLabels(r -> r.filters(f -> f.prefix("emr-7"))).releaseLabels())
+ .containsExactly("emr-7.5.0", "emr-7.4.0");
+ DescribeReleaseLabelResponse release = emr.describeReleaseLabel(r -> r.releaseLabel("emr-6.15.0"));
+ assertThat(release.applications())
+ .anySatisfy(a -> {
+ assertThat(a.name()).isEqualTo("spark");
+ assertThat(a.version()).isEqualTo("3.4.1");
+ });
+ assertThat(release.availableOSReleases()).isNotEmpty();
+ assertThat(emr.listSupportedInstanceTypes(r -> r.releaseLabel("emr-7.5.0")).supportedInstanceTypes())
+ .anySatisfy(t -> {
+ assertThat(t.type()).isEqualTo("r5.xlarge");
+ assertThat(t.vcpu()).isEqualTo(4);
+ assertThat(t.memoryGB()).isEqualTo(32f);
+ });
+ assertThatThrownBy(() -> emr.listSupportedInstanceTypes(r -> r.releaseLabel("emr-1.0.0")))
+ .isInstanceOf(InvalidRequestException.class);
+ }
}
diff --git a/docs/services/emr.md b/docs/services/emr.md
index fea52b099f..9598ba7e71 100644
--- a/docs/services/emr.md
+++ b/docs/services/emr.md
@@ -46,10 +46,31 @@ Floci emulates the Amazon EMR (Elastic MapReduce) management API. Clusters (job
| `RemoveAutoTerminationPolicy` | Removes a cluster's auto termination policy |
| `PutAutoScalingPolicy` | Attaches an automatic scaling policy to a core or task instance group |
| `RemoveAutoScalingPolicy` | Removes an instance group's automatic scaling policy |
+| `ListReleaseLabels` | List release labels, newest first, by prefix or application; see [Release catalogue](#release-catalogue) |
+| `DescribeReleaseLabel` | A release's applications and Amazon Linux releases (the latest release without a label) |
+| `ListSupportedInstanceTypes` | The instance types a release supports, with vCPU, memory, storage and architecture |
| `GetBlockPublicAccessConfiguration` | Returns the account's block public access configuration for the Region |
| `PutBlockPublicAccessConfiguration` | Sets the account's block public access configuration for the Region |
<!-- floci:actions:end -->
+## Release catalogue
+
+`ListReleaseLabels`, `DescribeReleaseLabel` and `ListSupportedInstanceTypes` read a curated catalogue,
+not AWS's full list: `emr-7.5.0` (Floci's default), `emr-7.4.0`, `emr-6.15.0`, `emr-6.14.0` and
+`emr-5.36.2`. Each release's applications and versions come from its page in the Amazon EMR Release
+Guide, with names lowercased and AWS's `-amzn-N` build suffix dropped, as the API reports them, and
+its two most recent Amazon Linux releases. A `floci.services.emr.default-release-label` outside the
+catalogue is still listed, with no applications.
+
+`ListReleaseLabels` filters by `Prefix` (such as `emr-6`) and `Application` (`spark` or
+`[email protected]`, matched without regard to case) and pages with `MaxResults` (1 to 100) /
+`NextToken`; a token only works with the filters it was issued for. `DescribeReleaseLabel` answers
+every application at once, since AWS reserves its `MaxResults` and `NextToken` for future use.
+`ListSupportedInstanceTypes` reports common families (c5, m5, m5d, m6g, r5, r6g) with
+their EC2 sizes. An unknown release label, a bad token or an out-of-range `MaxResults` is
+`InvalidRequestException`. `RunJobFlow` does not check its release label or instance types against
+the catalogue.
+
## Example
```bash
diff --git a/docs/services/index.md b/docs/services/index.md
index ba0ef4de03..40c55eea55 100644
--- a/docs/services/index.md
+++ b/docs/services/index.md
@@ -78,7 +78,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac
| [Redshift](redshift.md) | `POST /` with `Action=` param + PostgreSQL container | Query + PostgreSQL wire (+ CFN) | 43 |
| [Redshift Data API](redshift-data.md) | `POST /` + `X-Amz-Target: RedshiftData.*` | JSON 1.1 | 11 |
| [Redshift Serverless](redshift-serverless.md) | `POST /` + `X-Amz-Target: RedshiftServerless.*` | JSON 1.1 | 55 |
-| [EMR](emr.md) | `POST /` + `X-Amz-Target: ElasticMapReduce.*` | JSON 1.1 | 37 |
+| [EMR](emr.md) | `POST /` + `X-Amz-Target: ElasticMapReduce.*` | JSON 1.1 | 40 |
| [EMR Serverless](emr-serverless.md) | `/applications/*`, `/tags/*` | REST JSON | 10 |
| [Data Firehose](firehose.md) | `POST /` + `X-Amz-Target: Firehose_20150804.*` | JSON 1.1 | 12 |
| [ECS](ecs.md) | `POST /` + `X-Amz-Target: AmazonEC2ContainerServiceV20141113.*` | JSON 1.1 | 58 |
diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java b/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java
index d18c8d9c68..f4f10f554a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java
@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.core.common.AwsErrorResponse;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.AwsJson11Controller;
import io.github.hectorvent.floci.core.common.AwsRegions;
+import io.github.hectorvent.floci.core.common.PaginatedResult;
import io.github.hectorvent.floci.services.emr.model.EmrBlockPublicAccess;
import io.github.hectorvent.floci.services.emr.model.EmrCluster;
import io.github.hectorvent.floci.services.emr.model.EmrInstanceFleet;
@@ -83,6 +84,9 @@ public class EmrHandler {
case "RemoveAutoTerminationPolicy" -> handleRemoveAutoTerminationPolicy(request);
case "PutAutoScalingPolicy" -> handlePutAutoScalingPolicy(request);
case "RemoveAutoScalingPolicy" -> handleRemoveAutoScalingPolicy(request);
+ case "ListReleaseLabels" -> handleListReleaseLabels(request);
+ case "DescribeReleaseLabel" -> handleDescribeReleaseLabel(request);
+ case "ListSupportedInstanceTypes" -> handleListSupportedInstanceTypes(request);
case "GetBlockPublicAccessConfiguration" -> handleGetBlockPublicAccessConfiguration(region);
case "PutBlockPublicAccessConfiguration" -> handlePutBlockPublicAccessConfiguration(request, region);
default -> Response.status(400)
@@ -329,6 +333,71 @@ public class EmrHandler {
return Response.ok(objectMapper.createObjectNode()).build();
}
+ // ──────────────────────────── Release catalogue ────────────────────────────
+
+ private Response handleListReleaseLabels(JsonNode request) {
+ JsonNode filters = request.path("Filters");
+ PaginatedResult<String> page = service.listReleaseLabels(
+ filters.path("Prefix").asText(null), filters.path("Application").asText(null),
+ request.path("NextToken").asText(null), intOrNull(request.path("MaxResults")));
+ ObjectNode response = objectMapper.createObjectNode();
+ ArrayNode labels = response.putArray("ReleaseLabels");
+ page.items().forEach(labels::add);
+ if (page.nextToken() != null) {
+ response.put("NextToken", page.nextToken());
+ }
+ return Response.ok(response).build();
+ }
+
+ private Response handleDescribeReleaseLabel(JsonNode request) {
+ // MaxResults and NextToken are "reserved for future use" and AWS answers every application.
+ EmrReleaseCatalog.Release release = service.describeReleaseLabel(request.path("ReleaseLabel").asText(null));
+ ObjectNode response = objectMapper.createObjectNode();
+ response.put("ReleaseLabel", release.label());
+ ArrayNode apps = response.putArray("Applications");
+ for (EmrReleaseCatalog.Application app : release.applications()) {
+ apps.addObject().put("Name", app.name()).put("Version", app.version());
+ }
+ ArrayNode os = response.putArray("AvailableOSReleases");
+ release.osReleases().forEach(label -> os.addObject().put("Label", label));
+ return Response.ok(response).build();
+ }
+
+ private Response handleListSupportedInstanceTypes(JsonNode request) {
+ PaginatedResult<EmrReleaseCatalog.InstanceType> page = service.listSupportedInstanceTypes(
+ request.path("ReleaseLabel").asText(null), request.path("Marker").asText(null));
+ ObjectNode response = objectMapper.createObjectNode();
+ ArrayNode types = response.putArray("SupportedInstanceTypes");
+ for (EmrReleaseCatalog.InstanceType t : page.items()) {
+ types.addObject()
+ .put("Type", t.type())
+ .put("MemoryGB", t.memoryGb())
+ .put("StorageGB", t.storageGb())
+ .put("VCPU", t.vcpu())
+ .put("Is64BitsOnly", true)
+ .put("InstanceFamilyId", t.instanceFamilyId())
+ .put("EbsOptimizedAvailable", true)
+ .put("EbsOptimizedByDefault", true)
+ .put("NumberOfDisks", t.numberOfDisks())
+ .put("EbsStorageOnly", t.ebsStorageOnly())
+ .put("Architecture", t.architecture());
+ }
+ if (page.nextToken() != null) {
+ response.put("Marker", page.nextToken());
+ }
+ return Response.ok(response).build();
+ }
+
+ private static Integer intOrNull(JsonNode node) {
+ if (node == null || node.isMissingNode() || node.isNull()) {
+ return null;
+ }
+ if (!node.canConvertToInt()) {
+ throw new AwsException("InvalidRequestException", "MaxResults must be an integer.", 400);
+ }
+ return node.asInt();
+ }
+
private Response handleListSecurityConfigurations() {
ObjectNode response = objectMapper.createObjectNode();
ArrayNode arr = response.putArray("SecurityConfigurations");
diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/EmrReleaseCatalog.java b/src/main/java/io/github/hectorvent/floci/services/emr/EmrReleaseCatalog.java
new file mode 100644
index 0000000000..d39f637a09
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/emr/EmrReleaseCatalog.java
@@ -0,0 +1,207 @@
+package io.github.hectorvent.floci.services.emr;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * The EMR releases ListReleaseLabels, DescribeReleaseLabel and ListSupportedInstanceTypes report: a
+ * curated set of recent releases, not AWS's full list. Each release's applications and versions come
+ * from its page in the Amazon EMR Release Guide (application names lowercased and versions without
+ * AWS's {@code -amzn-N} build suffix, as the API reports them), with the two most recent Amazon Linux
+ * releases that page lists. The instance types are common EMR families with their EC2 sizes, all of
+ * which these releases support.
+ */
+final class EmrReleaseCatalog {
+
+ record Application(String name, String version) {}
+
+ record Release(String label, List<Application> applications, List<String> osReleases) {}
+
+ record InstanceType(String type, int vcpu, float memoryGb, int storageGb, int numberOfDisks,
+ boolean ebsStorageOnly, String instanceFamilyId, String architecture) {}
+
+ /** Newest first, as ListReleaseLabels returns them. */
+ static final List<Release> RELEASES = List.of(
+ new Release("emr-7.5.0", List.of(
+ app("amazoncloudwatchagent", "1.300032.2"),
+ app("delta", "3.2.1"),
+ app("flink", "1.19.1"),
+ app("hadoop", "3.4.0"),
+ app("hbase", "2.5.10"),
+ app("hcatalog", "3.1.3"),
+ app("hive", "3.1.3"),
+ app("hudi", "0.15.0"),
+ app("hue", "4.11.0"),
+ app("iceberg", "1.6.1"),
+ app("jupyterenterprisegateway", "2.6.0"),
+ app("jupyterhub", "1.5.0"),
+ app("livy", "0.8.0-incubating"),
+ app("oozie", "5.2.1"),
+ app("phoenix", "5.2.0"),
+ app("pig", "0.17.0"),
+ app("presto", "0.287"),
+ app("spark", "3.5.2"),
+ app("tensorflow", "2.16.1"),
+ app("tez", "0.10.2"),
+ app("trino", "446"),
+ app("zeppelin", "0.11.1"),
+ app("zookeeper", "3.9.2")),
+ List.of("2023.12.20260803.3", "2023.12.20260727.0")),
+ new Release("emr-7.4.0", List.of(
+ app("amazoncloudwatchagent", "1.300032.2"),
+ app("delta", "3.2.0"),
+ app("flink", "1.19.1"),
+ app("hadoop", "3.4.0"),
+ app("hbase", "2.5.5"),
+ app("hcatalog", "3.1.3"),
+ app("hive", "3.1.3"),
+ app("hudi", "0.15.0"),
+ app("hue", "4.11.0"),
+ app("iceberg", "1.6.1"),
+ app("jupyterenterprisegateway", "2.6.0"),
+ app("jupyterhub", "1.5.0"),
+ app("livy", "0.8.0-incubating"),
+ app("oozie", "5.2.1"),
+ app("phoenix", "5.2.0"),
+ app("pig", "0.17.0"),
+ app("presto", "0.287"),
+ app("spark", "3.5.2"),
+ app("sqoop", "1.4.7"),
+ app("tensorflow", "2.16.1"),
+ app("tez", "0.10.2"),
+ app("trino", "446"),
+ app("zeppelin", "0.11.1"),
+ app("zookeeper", "3.9.2")),
+ List.of("2023.12.20260803.3", "2023.12.20260727.0")),
+ new Release("emr-6.15.0", List.of(
+ app("delta", "2.4.0"),
+ app("flink", "1.17.1"),
+ app("ganglia", "3.7.2"),
+ app("hadoop", "3.3.6"),
+ app("hbase", "2.4.17"),
+ app("hcatalog", "3.1.3"),
+ app("hive", "3.1.3"),
+ app("hudi", "0.14.0"),
+ app("hue", "4.11.0"),
+ app("iceberg", "1.4.0"),
+ app("jupyterenterprisegateway", "2.6.0"),
+ app("jupyterhub", "1.5.0"),
+ app("livy", "0.7.1-incubating"),
+ app("mxnet", "1.9.1"),
+ app("oozie", "5.2.1"),
+ app("phoenix", "5.1.3"),
+ app("pig", "0.17.0"),
+ app("presto", "0.283"),
+ app("spark", "3.4.1"),
+ app("sqoop", "1.4.7"),
+ app("tensorflow", "2.11.0"),
+ app("tez", "0.10.2"),
+ app("trino", "426"),
+ app("zeppelin", "0.10.1"),
+ app("zookeeper", "3.5.10")),
+ List.of("2.0.20260803.1", "2.0.20260727.0")),
+ new Release("emr-6.14.0", List.of(
+ app("delta", "2.4.0"),
+ app("flink", "1.17.1"),
+ app("ganglia", "3.7.2"),
+ app("hadoop", "3.3.3"),
+ app("hbase", "2.4.17"),
+ app("hcatalog", "3.1.3"),
+ app("hive", "3.1.3"),
+ app("hudi", "0.13.1"),
+ app("hue", "4.11.0"),
+ app("iceberg", "1.3.1"),
+ app("jupyterenterprisegateway", "2.6.0"),
+ app("jupyterhub", "1.5.0"),
+ app("livy", "0.7.1-incubating"),
+ app("mxnet", "1.9.1"),
+ app("oozie", "5.2.1"),
+ app("phoenix", "5.1.3"),
+ app("pig", "0.17.0"),
+ app("presto", "0.281"),
+ app("spark", "3.4.1"),
+ app("sqoop", "1.4.7"),
+ app("tensorflow", "2.11.0"),
+ app("tez", "0.10.2"),
+ app("trino", "422"),
+ app("zeppelin", "0.10.1"),
+ app("zookeeper", "3.5.10")),
+ List.of("2.0.20260803.1", "2.0.20260727.0")),
+ new Release("emr-5.36.2", List.of(
+ app("flink", "1.14.2"),
+ app("ganglia", "3.7.2"),
+ app("hadoop", "2.10.1"),
+ app("hbase", "1.4.13"),
+ app("hcatalog", "2.3.9"),
+ app("hive", "2.3.9"),
+ app("hudi", "0.10.1"),
+ app("hue", "4.10.0"),
+ app("jupyterenterprisegateway", "2.6.0"),
+ app("jupyterhub", "1.4.1"),
+ app("livy", "0.7.1-incubating"),
+ app("mahout", "0.13.0"),
+ app("mxnet", "1.8.0"),
+ app("oozie", "5.2.1"),
+ app("phoenix", "4.14.3-HBase-1.4"),
+ app("pig", "0.17.0"),
+ app("presto", "0.267"),
+ app("spark", "2.4.8"),
+ app("sqoop", "1.4.7"),
+ app("tensorflow", "2.4.1"),
+ app("tez", "0.9.2"),
+ app("zeppelin", "0.10.0"),
+ app("zookeeper", "3.4.14")),
+ List.of("2.0.20260803.1", "2.0.20260727.0")));
+
+ static final List<InstanceType> INSTANCE_TYPES = List.of(
+ new InstanceType("c5.xlarge", 4, 8f, 0, 0, true, "c5", "X86_64"),
+ new InstanceType("c5.2xlarge", 8, 16f, 0, 0, true, "c5", "X86_64"),
+ new InstanceType("m5.xlarge", 4, 16f, 0, 0, true, "m5", "X86_64"),
+ new InstanceType("m5.2xlarge", 8, 32f, 0, 0, true, "m5", "X86_64"),
+ new InstanceType("m5.4xlarge", 16, 64f, 0, 0, true, "m5", "X86_64"),
+ new InstanceType("m5d.xlarge", 4, 16f, 150, 1, false, "m5d", "X86_64"),
+ new InstanceType("m5d.2xlarge", 8, 32f, 300, 1, false, "m5d", "X86_64"),
+ new InstanceType("m6g.xlarge", 4, 16f, 0, 0, true, "m6g", "AARCH64"),
+ new InstanceType("m6g.2xlarge", 8, 32f, 0, 0, true, "m6g", "AARCH64"),
+ new InstanceType("r5.xlarge", 4, 32f, 0, 0, true, "r5", "X86_64"),
+ new InstanceType("r5.2xlarge", 8, 64f, 0, 0, true, "r5", "X86_64"),
+ new InstanceType("r6g.xlarge", 4, 32f, 0, 0, true, "r6g", "AARCH64"));
+
+ private EmrReleaseCatalog() {
+ }
+
+ private static Application app(String name, String version) {
+ return new Application(name, version);
+ }
+
+ static Optional<Release> release(String label) {
+ return RELEASES.stream().filter(r -> r.label().equals(label)).findFirst();
+ }
+
+ /** Numeric release order, so emr-6.15.0 sorts after emr-6.9.0. */
+ static final Comparator<String> NEWEST_FIRST = Comparator.comparing(EmrReleaseCatalog::versionKey,
+ (a, b) -> {
+ for (int i = 0; i < Math.max(a.length, b.length); i++) {
+ int x = i < a.length ? a[i] : 0;
+ int y = i < b.length ? b[i] : 0;
+ if (x != y) {
+ return Integer.compare(y, x);
+ }
+ }
+ return 0;
+ });
+
+ private static int[] versionKey(String label) {
+ String[] parts = label.replaceFirst("^emr-", "").split("\\.");
+ int[] key = new int[parts.length];
+ for (int i = 0; i < parts.length; i++) {
+ try {
+ key[i] = Integer.parseInt(parts[i]);
+ } catch (NumberFormatException e) {
+ key[i] = 0;
+ }
+ }
+ return key;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java b/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java
index 3546596d79..fb55cd4e53 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java
@@ -5,6 +5,8 @@ import com.fasterxml.jackson.databind.JsonNode;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.AwsRegions;
+import io.github.hectorvent.floci.core.common.PaginatedResult;
+import io.github.hectorvent.floci.core.common.Pagination;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.SmithyValidationErrors;
import io.github.hectorvent.floci.core.storage.StorageBackend;
@@ -21,11 +23,14 @@ import jakarta.inject.Inject;
import java.security.SecureRandom;
import java.time.Instant;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import java.util.function.Function;
/**
* EMR management-plane business logic. Cluster and step lifecycles are simulated
@@ -402,6 +407,97 @@ public class EmrService {
secConfigStore.delete(name);
}
+ // ──────────────────────────── Release catalogue ────────────────────────────
+
+ private static final int MAX_RESULTS = 100;
+ private static final int INSTANCE_TYPE_PAGE = 50;
+
+ /**
+ * The catalogue's releases, newest first. Floci's configured default release label is always
+ * among them, with no applications when the catalogue does not describe it, so a non-default
+ * setting never makes Floci's own default unknown.
+ */
+ List<EmrReleaseCatalog.Release> releases() {
+ if (EmrReleaseCatalog.release(defaultReleaseLabel).isPresent()) {
+ return EmrReleaseCatalog.RELEASES;
+ }
+ List<EmrReleaseCatalog.Release> all = new ArrayList<>(EmrReleaseCatalog.RELEASES);
+ all.add(new EmrReleaseCatalog.Release(defaultReleaseLabel, List.of(), List.of()));
+ all.sort((a, b) -> EmrReleaseCatalog.NEWEST_FIRST.compare(a.label(), b.label()));
+ return all;
+ }
+
+ /**
+ * Release labels, newest first, narrowed by {@code prefix} (such as {@code emr-6}) and by
+ * {@code application}, a name or {@code name@version} (such as {@code [email protected]}) matched
+ * without regard to case. A token belongs to the filters it was issued for, so one replayed
+ * with other filters, or altered, is refused.
+ */
+ public PaginatedResult<String> listReleaseLabels(String prefix, String application, String nextToken,
+ Integer maxResults) {
+ if (maxResults != null && (maxResults < 1 || maxResults > MAX_RESULTS)) {
+ throw new AwsException("InvalidRequestException",
+ "MaxResults must be between 1 and " + MAX_RESULTS + ".", 400);
+ }
+ String appName = null;
+ String appVersion = null;
+ if (application != null && !application.isBlank()) {
+ int at = application.indexOf('@');
+ appName = (at < 0 ? application : application.substring(0, at)).trim();
+ appVersion = at < 0 ? null : application.substring(at + 1).trim();
+ }
+ String name = appName;
+ String version = appVersion;
+ List<String> labels = releases().stream()
+ .filter(r -> prefix == null || prefix.isBlank() || r.label().startsWith(prefix))
+ .filter(r -> name == null || r.applications().stream().anyMatch(a -> a.name().equalsIgnoreCase(name)
+ && (version == null || version.isEmpty() || a.version().equalsIgnoreCase(version))))
+ .map(EmrReleaseCatalog.Release::label)
+ .toList();
+ String namespace = "ListReleaseLabels|" + Objects.toString(prefix, "") + "|" + Objects.toString(application, "");
+ return Pagination.paginate(labels, positionIn(labels), maxResults != null ? maxResults : MAX_RESULTS,
+ nextToken, namespace, EmrService::invalidToken);
+ }
+
+ /** A release's applications and OS releases; no label means the latest release. */
+ public EmrReleaseCatalog.Release describeReleaseLabel(String label) {
+ return label == null || label.isBlank() ? releases().get(0) : requireRelease(label);
+ }
+
+ public PaginatedResult<EmrReleaseCatalog.InstanceType> listSupportedInstanceTypes(String label, String marker) {
+ if (label == null || label.isBlank()) {
+ throw new AwsException("InvalidRequestException", "ReleaseLabel is required.", 400);
+ }
+ requireRelease(label);
+ List<EmrReleaseCatalog.InstanceType> types = EmrReleaseCatalog.INSTANCE_TYPES;
+ return Pagination.paginate(types, positionIn(types), INSTANCE_TYPE_PAGE, marker,
+ "ListSupportedInstanceTypes|" + label, EmrService::invalidToken);
+ }
+
+ private EmrReleaseCatalog.Release requireRelease(String label) {
+ return releases().stream()
+ .filter(r -> r.label().equals(label))
+ .findFirst()
+ .orElseThrow(() -> new AwsException("InvalidRequestException",
+ "The specified release label " + label + " is not valid.", 400));
+ }
+
+ /**
+ * A cursor that keeps the catalogue's own order: {@link Pagination} sorts by cursor, and the
+ * catalogue is newest first rather than alphabetical.
+ */
+ private static <T> Function<T, String> positionIn(List<T> items) {
+ Map<T, String> positions = new HashMap<>();
+ for (int i = 0; i < items.size(); i++) {
+ positions.put(items.get(i), String.format("%06d", i));
+ }
+ return positions::get;
+ }
+
+ private static AwsException invalidToken(String token) {
+ return new AwsException("InvalidRequestException", "Invalid pagination token: " + token, 400);
+ }
+
public List<SecurityConfiguration> listSecurityConfigurations() {
return new ArrayList<>(secConfigStore.scan(k -> true));
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emr/EmrClusterRegionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emr/EmrClusterRegionIntegrationTest.java
index db0f03bb5f..6fbf8895d8 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emr/EmrClusterRegionIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emr/EmrClusterRegionIntegrationTest.java
@@ -2,11 +2,10 @@ package io.github.hectorvent.floci.services.emr;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
-import io.restassured.response.Response;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
-import static io.restassured.RestAssured.given;
+import static io.github.hectorvent.floci.testing.EmrRequests.emr;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.not;
@@ -18,8 +17,6 @@ import static org.hamcrest.Matchers.not;
@QuarkusTest
class EmrClusterRegionIntegrationTest {
- private static final String CT = "application/x-amz-json-1.1";
- private static final String PREFIX = "ElasticMapReduce.";
private static final String HOME = "eu-west-1";
private static final String OTHER = "us-west-2";
@@ -28,17 +25,9 @@ class EmrClusterRegionIntegrationTest {
RestAssuredJsonUtils.configureAwsContentTypes();
}
- private static Response call(String region, String action, String body) {
- return given().contentType(CT)
- .header("Authorization", "AWS4-HMAC-SHA256 Credential=AKID/20260215/" + region
- + "/elasticmapreduce/aws4_request, SignedHeaders=host, Signature=abc")
- .header("X-Amz-Target", PREFIX + action)
- .body(body).when().post("/");
- }
-
@Test
void clusterOperationsStayInTheClustersRegion() {
- String clusterId = call(HOME, "RunJobFlow",
+ String clusterId = emr(HOME, "RunJobFlow",
"{\"Name\":\"region-emr\",\"ReleaseLabel\":\"emr-7.5.0\","
+ "\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true,"
+ "\"InstanceGroups\":[{\"Name\":\"master\",\"InstanceRole\":\"MASTER\","
@@ -47,34 +36,34 @@ class EmrClusterRegionIntegrationTest {
.extract().path("JobFlowId");
String byId = "{\"ClusterId\":\"" + clusterId + "\"}";
- call(OTHER, "DescribeCluster", byId)
+ emr(OTHER, "DescribeCluster", byId)
.then().statusCode(400)
.body("__type", equalTo("InvalidRequestException"));
- call(OTHER, "AddJobFlowSteps", "{\"JobFlowId\":\"" + clusterId + "\",\"Steps\":[{\"Name\":\"s\","
+ emr(OTHER, "AddJobFlowSteps", "{\"JobFlowId\":\"" + clusterId + "\",\"Steps\":[{\"Name\":\"s\","
+ "\"HadoopJarStep\":{\"Jar\":\"command-runner.jar\",\"Args\":[\"echo\"]}}]}")
.then().statusCode(400)
.body("__type", equalTo("InvalidRequestException"));
- call(OTHER, "ListClusters", "{}")
+ emr(OTHER, "ListClusters", "{}")
.then().statusCode(200)
.body("Clusters.Id", not(hasItem(clusterId)));
- call(OTHER, "SetTerminationProtection",
+ emr(OTHER, "SetTerminationProtection",
"{\"JobFlowIds\":[\"" + clusterId + "\"],\"TerminationProtected\":true}")
.then().statusCode(200);
- call(OTHER, "TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}")
+ emr(OTHER, "TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}")
.then().statusCode(200);
- call(HOME, "DescribeCluster", byId)
+ emr(HOME, "DescribeCluster", byId)
.then().statusCode(200)
.body("Cluster.Status.State", equalTo("WAITING"))
.body("Cluster.TerminationProtected", equalTo(false));
- call(HOME, "TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}")
+ emr(HOME, "TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}")
.then().statusCode(200);
}
// Catches: an instance's or the master's private DNS name in the wrong region's domain.
@Test
void listedInstancesUseTheClustersRegionalPrivateDnsDomain() {
- String clusterId = call(HOME, "RunJobFlow",
+ String clusterId = emr(HOME, "RunJobFlow",
"{\"Name\":\"region-dns\",\"ReleaseLabel\":\"emr-7.5.0\","
+ "\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true,"
+ "\"InstanceGroups\":[{\"Name\":\"master\",\"InstanceRole\":\"MASTER\","
@@ -82,15 +71,15 @@ class EmrClusterRegionIntegrationTest {
.then().statusCode(200)
.extract().path("JobFlowId");
- call(HOME, "ListInstances", "{\"ClusterId\":\"" + clusterId + "\"}")
+ emr(HOME, "ListInstances", "{\"ClusterId\":\"" + clusterId + "\"}")
.then().statusCode(200)
.body("Instances[0].PrivateDnsName", equalTo("ip-10-0-0-1.eu-west-1.compute.internal"))
.body("Instances[0].PrivateIpAddress", equalTo("10.0.0.1"));
- call(HOME, "DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
+ emr(HOME, "DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
.then().statusCode(200)
.body("Cluster.MasterPublicDnsName", equalTo("ip-10-0-0-1.eu-west-1.compute.internal"));
- call(HOME, "TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}")
+ emr(HOME, "TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}")
.then().statusCode(200);
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java
index a3723d5e3c..7e7401b04b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java
@@ -10,7 +10,7 @@ import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
-import static io.restassured.RestAssured.given;
+import static io.github.hectorvent.floci.testing.EmrRequests.emr;
import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
@@ -26,9 +26,6 @@ import static org.hamcrest.Matchers.startsWith;
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
class EmrIntegrationTest {
- private static final String CT = "application/x-amz-json-1.1";
- private static final String PREFIX = "ElasticMapReduce.";
-
private static String clusterId;
private static String stepId;
@@ -40,15 +37,10 @@ class EmrIntegrationTest {
RestAssuredJsonUtils.configureAwsContentTypes();
}
- private static Response call(String action, String body) {
- return given().contentType(CT).header("X-Amz-Target", PREFIX + action)
- .body(body).when().post("/");
- }
-
@Test
@Order(1)
void runJobFlow() {
- Response resp = call("RunJobFlow",
+ Response resp = emr("RunJobFlow",
"{\"Name\":\"floci-emr\",\"ReleaseLabel\":\"emr-7.5.0\","
+ "\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true,"
+ "\"InstanceGroups\":[{\"Name\":\"master\",\"InstanceRole\":\"MASTER\","
@@ -66,7 +58,7 @@ class EmrIntegrationTest {
@Test
@Order(2)
void describeClusterIsWaiting() {
- call("DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
+ emr("DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
.then().statusCode(200)
.body("Cluster.Id", equalTo(clusterId))
.body("Cluster.Status.State", equalTo("WAITING"))
@@ -79,7 +71,7 @@ class EmrIntegrationTest {
@Test
@Order(3)
void listClustersFilteredByState() {
- call("ListClusters", "{\"ClusterStates\":[\"WAITING\"]}")
+ emr("ListClusters", "{\"ClusterStates\":[\"WAITING\"]}")
.then().statusCode(200)
.body("Clusters.find { it.Id == '" + clusterId + "' }.Name", equalTo("floci-emr"));
}
@@ -87,7 +79,7 @@ class EmrIntegrationTest {
@Test
@Order(4)
void listInstanceGroups() {
- call("ListInstanceGroups", "{\"ClusterId\":\"" + clusterId + "\"}")
+ emr("ListInstanceGroups", "{\"ClusterId\":\"" + clusterId + "\"}")
.then().statusCode(200)
.body("InstanceGroups", hasSize(2))
.body("InstanceGroups.find { it.InstanceGroupType == 'CORE' }.RunningInstanceCount", equalTo(2));
@@ -96,7 +88,7 @@ class EmrIntegrationTest {
@Test
@Order(5)
void listInstancesSynthetic() {
- call("ListInstances", "{\"ClusterId\":\"" + clusterId + "\"}")
+ emr("ListInstances", "{\"ClusterId\":\"" + clusterId + "\"}")
.then().statusCode(200)
.body("Instances", hasSize(3)) // 1 master + 2 core
.body("Instances[0].Status.State", equalTo("RUNNING"))
@@ -106,7 +98,7 @@ class EmrIntegrationTest {
@Test
@Order(6)
void addStepWithLegacyActionAlias() {
- stepId = call("AddJobFlowSteps",
+ stepId = emr("AddJobFlowSteps",
"{\"JobFlowId\":\"" + clusterId + "\",\"Steps\":[{\"Name\":\"added\","
+ "\"ActionOnFailure\":\"TERMINATE_CLUSTER\","
+ "\"HadoopJarStep\":{\"Jar\":\"command-runner.jar\",\"MainClass\":\"org.Main\","
@@ -119,7 +111,7 @@ class EmrIntegrationTest {
@Test
@Order(7)
void describeStepCompleted() {
- call("DescribeStep", "{\"ClusterId\":\"" + clusterId + "\",\"StepId\":\"" + stepId + "\"}")
+ emr("DescribeStep", "{\"ClusterId\":\"" + clusterId + "\",\"StepId\":\"" + stepId + "\"}")
.then().statusCode(200)
.body("Step.Id", equalTo(stepId))
.body("Step.Name", equalTo("added"))
@@ -131,7 +123,7 @@ class EmrIntegrationTest {
@Test
@Order(8)
void listStepsNewestFirst() {
- call("ListSteps", "{\"ClusterId\":\"" + clusterId + "\"}")
+ emr("ListSteps", "{\"ClusterId\":\"" + clusterId + "\"}")
.then().statusCode(200)
.body("Steps", hasSize(2))
.body("Steps[0].Name", equalTo("added")) // newest first
@@ -141,7 +133,7 @@ class EmrIntegrationTest {
@Test
@Order(9)
void modifyStepConcurrency() {
- call("ModifyCluster", "{\"ClusterId\":\"" + clusterId + "\",\"StepConcurrencyLevel\":5}")
+ emr("ModifyCluster", "{\"ClusterId\":\"" + clusterId + "\",\"StepConcurrencyLevel\":5}")
.then().statusCode(200)
.body("StepConcurrencyLevel", equalTo(5));
}
@@ -149,36 +141,36 @@ class EmrIntegrationTest {
@Test
@Order(10)
void securityConfigurationRoundTrip() {
- call("CreateSecurityConfiguration",
+ emr("CreateSecurityConfiguration",
"{\"Name\":\"floci-emr-sec\",\"SecurityConfiguration\":\"{\\\"EncryptionConfiguration\\\":{}}\"}")
.then().statusCode(200)
.body("Name", equalTo("floci-emr-sec"))
.body("CreationDateTime", notNullValue());
- call("DescribeSecurityConfiguration", "{\"Name\":\"floci-emr-sec\"}")
+ emr("DescribeSecurityConfiguration", "{\"Name\":\"floci-emr-sec\"}")
.then().statusCode(200)
.body("SecurityConfiguration", equalTo("{\"EncryptionConfiguration\":{}}"));
- call("DeleteSecurityConfiguration", "{\"Name\":\"floci-emr-sec\"}").then().statusCode(200);
+ emr("DeleteSecurityConfiguration", "{\"Name\":\"floci-emr-sec\"}").then().statusCode(200);
}
@Test
@Order(11)
void tagsRoundTrip() {
- call("AddTags", "{\"ResourceId\":\"" + clusterId + "\",\"Tags\":[{\"Key\":\"env\",\"Value\":\"test\"}]}")
+ emr("AddTags", "{\"ResourceId\":\"" + clusterId + "\",\"Tags\":[{\"Key\":\"env\",\"Value\":\"test\"}]}")
.then().statusCode(200);
- call("DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
+ emr("DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
.then().statusCode(200)
.body("Cluster.Tags.find { it.Key == 'env' }.Value", equalTo("test"));
- call("RemoveTags", "{\"ResourceId\":\"" + clusterId + "\",\"TagKeys\":[\"env\"]}")
+ emr("RemoveTags", "{\"ResourceId\":\"" + clusterId + "\",\"TagKeys\":[\"env\"]}")
.then().statusCode(200);
}
@Test
@Order(12)
void terminateCluster() {
- call("TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}").then().statusCode(200);
- call("DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
+ emr("TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}").then().statusCode(200);
+ emr("DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
.then().statusCode(200)
.body("Cluster.Status.State", equalTo("TERMINATED"))
.body("Cluster.Status.StateChangeReason.Code", equalTo("USER_REQUEST"));
@@ -187,7 +179,7 @@ class EmrIntegrationTest {
@Test
@Order(13)
void describeUnknownClusterIsInvalidRequest() {
- call("DescribeCluster", "{\"ClusterId\":\"j-DOESNOTEXIST0\"}")
+ emr("DescribeCluster", "{\"ClusterId\":\"j-DOESNOTEXIST0\"}")
.then().statusCode(400)
.body("__type", equalTo("InvalidRequestException"));
}
@@ -195,7 +187,7 @@ class EmrIntegrationTest {
@Test
@Order(14)
void runJobFlowWithoutNameIsValidationException() {
- call("RunJobFlow", "{\"ReleaseLabel\":\"emr-7.5.0\",\"Instances\":{}}")
+ emr("RunJobFlow", "{\"ReleaseLabel\":\"emr-7.5.0\",\"Instances\":{}}")
.then().statusCode(400)
.body("__type", equalTo("ValidationException"))
.body("message", equalTo(
@@ -208,7 +200,7 @@ class EmrIntegrationTest {
void runJobFlowWithEmptyNameIsAccepted() {
// The Name member is REQUIRED but its model constraint is len=[0..256], so real EMR
// accepts an explicit empty string; only a missing Name fails framework validation.
- call("RunJobFlow", "{\"Name\":\"\",\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true}}")
+ emr("RunJobFlow", "{\"Name\":\"\",\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true}}")
.then().statusCode(200)
.body("JobFlowId", startsWith("j-"));
}
@@ -216,7 +208,7 @@ class EmrIntegrationTest {
@Test
@Order(16)
void addStepsToUnknownClusterIsInvalidRequest() {
- call("AddJobFlowSteps",
+ emr("AddJobFlowSteps",
"{\"JobFlowId\":\"j-DOESNOTEXIST0\",\"Steps\":[{\"Name\":\"s\","
+ "\"HadoopJarStep\":{\"Jar\":\"command-runner.jar\"}}]}")
.then().statusCode(400)
@@ -227,28 +219,28 @@ class EmrIntegrationTest {
@Test
@Order(17)
void terminationProtectionIsValidationExceptionAndUnprotectedStillTerminate() {
- String protectedId = call("RunJobFlow",
+ String protectedId = emr("RunJobFlow",
"{\"Name\":\"protected\",\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true}}")
.jsonPath().getString("JobFlowId");
- String unprotectedId = call("RunJobFlow",
+ String unprotectedId = emr("RunJobFlow",
"{\"Name\":\"unprotected\",\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true}}")
.jsonPath().getString("JobFlowId");
- call("SetTerminationProtection",
+ emr("SetTerminationProtection",
"{\"JobFlowIds\":[\"" + protectedId + "\"],\"TerminationProtected\":true}")
.then().statusCode(200);
// AWS terminates the unprotected clusters in the request and then fails it.
- call("TerminateJobFlows",
+ emr("TerminateJobFlows",
"{\"JobFlowIds\":[\"" + unprotectedId + "\",\"" + protectedId + "\"]}")
.then().statusCode(400)
.body("__type", equalTo("ValidationException"))
.body("message", equalTo(
"Could not shut down one or more job flows since they are termination protected."));
- call("DescribeCluster", "{\"ClusterId\":\"" + unprotectedId + "\"}")
+ emr("DescribeCluster", "{\"ClusterId\":\"" + unprotectedId + "\"}")
.then().statusCode(200)
.body("Cluster.Status.State", equalTo("TERMINATED"));
- call("DescribeCluster", "{\"ClusterId\":\"" + protectedId + "\"}")
+ emr("DescribeCluster", "{\"ClusterId\":\"" + protectedId + "\"}")
.then().statusCode(200)
.body("Cluster.Status.State", equalTo("WAITING"));
}
@@ -256,7 +248,7 @@ class EmrIntegrationTest {
// Catches: a cluster stored before the DNS fix keeps reporting its old us-east-1 master name.
@Test
void storedClusterReportsTheRegionalMasterDnsName() {
- String id = call("RunJobFlow",
+ String id = emr("RunJobFlow",
"{\"Name\":\"stale-dns\",\"ReleaseLabel\":\"emr-7.5.0\","
+ "\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true,"
+ "\"InstanceGroups\":[{\"Name\":\"master\",\"InstanceRole\":\"MASTER\","
@@ -265,11 +257,11 @@ class EmrIntegrationTest {
.extract().path("JobFlowId");
emrService.describeCluster(id).setMasterPublicDnsName("ip-10-0-0-1.us-east-1.compute.internal");
- call("DescribeCluster", "{\"ClusterId\":\"" + id + "\"}")
+ emr("DescribeCluster", "{\"ClusterId\":\"" + id + "\"}")
.then().statusCode(200)
.body("Cluster.MasterPublicDnsName", equalTo("ip-10-0-0-1.ec2.internal"));
- call("TerminateJobFlows", "{\"JobFlowIds\":[\"" + id + "\"]}")
+ emr("TerminateJobFlows", "{\"JobFlowIds\":[\"" + id + "\"]}")
.then().statusCode(200);
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emr/EmrPoliciesIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emr/EmrPoliciesIntegrationTest.java
index 6ba3d92793..39c61b883b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emr/EmrPoliciesIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emr/EmrPoliciesIntegrationTest.java
@@ -9,7 +9,7 @@ import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
-import static io.restassured.RestAssured.given;
+import static io.github.hectorvent.floci.testing.EmrRequests.emr;
import static org.hamcrest.Matchers.anEmptyMap;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
@@ -24,8 +24,6 @@ import static org.hamcrest.Matchers.nullValue;
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
class EmrPoliciesIntegrationTest {
- private static final String CT = "application/x-amz-json-1.1";
- private static final String PREFIX = "ElasticMapReduce.";
private static final String MANAGED_SCALING =
"{\"ComputeLimits\":{\"UnitType\":\"Instances\",\"MinimumCapacityUnits\":1,\"MaximumCapacityUnits\":5}}";
private static final String AUTO_SCALING =
@@ -43,11 +41,6 @@ class EmrPoliciesIntegrationTest {
RestAssuredJsonUtils.configureAwsContentTypes();
}
- private static Response call(String action, String body) {
- return given().contentType(CT).header("X-Amz-Target", PREFIX + action)
- .body(body).when().post("/");
- }
-
private static String cluster() {
return "{\"ClusterId\":\"" + clusterId + "\"";
}
@@ -55,7 +48,7 @@ class EmrPoliciesIntegrationTest {
@Test
@Order(1)
void runJobFlowStoresPolicies() {
- Response resp = call("RunJobFlow",
+ Response resp = emr("RunJobFlow",
"{\"Name\":\"policies\",\"ReleaseLabel\":\"emr-7.5.0\","
+ "\"ManagedScalingPolicy\":" + MANAGED_SCALING + ","
+ "\"AutoTerminationPolicy\":{\"IdleTimeout\":3600},"
@@ -66,11 +59,11 @@ class EmrPoliciesIntegrationTest {
resp.then().statusCode(200);
clusterId = resp.jsonPath().getString("JobFlowId");
- call("GetManagedScalingPolicy", cluster() + "}").then().statusCode(200)
+ emr("GetManagedScalingPolicy", cluster() + "}").then().statusCode(200)
.body("ManagedScalingPolicy.ComputeLimits.MaximumCapacityUnits", equalTo(5));
- call("GetAutoTerminationPolicy", cluster() + "}").then().statusCode(200)
+ emr("GetAutoTerminationPolicy", cluster() + "}").then().statusCode(200)
.body("AutoTerminationPolicy.IdleTimeout", equalTo(3600));
- Response groups = call("ListInstanceGroups", cluster() + "}");
+ Response groups = emr("ListInstanceGroups", cluster() + "}");
groups.then().statusCode(200)
.body("InstanceGroups.find { it.InstanceGroupType == 'CORE' }.AutoScalingPolicy.Status.State",
equalTo("ATTACHED"))
@@ -84,53 +77,53 @@ class EmrPoliciesIntegrationTest {
@Test
@Order(2)
void managedScalingPolicyPutGetRemove() {
- call("PutManagedScalingPolicy", cluster() + ",\"ManagedScalingPolicy\":{\"ComputeLimits\":"
+ emr("PutManagedScalingPolicy", cluster() + ",\"ManagedScalingPolicy\":{\"ComputeLimits\":"
+ "{\"UnitType\":\"VCPU\",\"MinimumCapacityUnits\":4,\"MaximumCapacityUnits\":32},"
+ "\"ScalingStrategy\":\"ADVANCED\"}}").then().statusCode(200);
- call("GetManagedScalingPolicy", cluster() + "}").then().statusCode(200)
+ emr("GetManagedScalingPolicy", cluster() + "}").then().statusCode(200)
.body("ManagedScalingPolicy.ComputeLimits.UnitType", equalTo("VCPU"))
.body("ManagedScalingPolicy.ScalingStrategy", equalTo("ADVANCED"));
- call("RemoveManagedScalingPolicy", cluster() + "}").then().statusCode(200);
- call("GetManagedScalingPolicy", cluster() + "}").then().statusCode(200)
+ emr("RemoveManagedScalingPolicy", cluster() + "}").then().statusCode(200);
+ emr("GetManagedScalingPolicy", cluster() + "}").then().statusCode(200)
.body("$", anEmptyMap());
}
@Test
@Order(3)
void managedScalingPolicyIsValidated() {
- call("PutManagedScalingPolicy", cluster() + ",\"ManagedScalingPolicy\":{}}")
+ emr("PutManagedScalingPolicy", cluster() + ",\"ManagedScalingPolicy\":{}}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("PutManagedScalingPolicy", cluster() + ",\"ManagedScalingPolicy\":{\"ComputeLimits\":"
+ emr("PutManagedScalingPolicy", cluster() + ",\"ManagedScalingPolicy\":{\"ComputeLimits\":"
+ "{\"UnitType\":\"Instances\",\"MinimumCapacityUnits\":6,\"MaximumCapacityUnits\":5}}}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("PutManagedScalingPolicy", cluster() + ",\"ManagedScalingPolicy\":{\"ComputeLimits\":"
+ emr("PutManagedScalingPolicy", cluster() + ",\"ManagedScalingPolicy\":{\"ComputeLimits\":"
+ "{\"UnitType\":\"Instances\",\"MinimumCapacityUnits\":1,\"MaximumCapacityUnits\":5,"
+ "\"MaximumOnDemandCapacityUnits\":-1}}}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("PutManagedScalingPolicy", "{\"ClusterId\":\"j-MISSING\",\"ManagedScalingPolicy\":" + MANAGED_SCALING + "}")
+ emr("PutManagedScalingPolicy", "{\"ClusterId\":\"j-MISSING\",\"ManagedScalingPolicy\":" + MANAGED_SCALING + "}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
}
@Test
@Order(4)
void autoTerminationPolicyPutGetRemove() {
- call("PutAutoTerminationPolicy", cluster() + ",\"AutoTerminationPolicy\":{\"IdleTimeout\":120}}")
+ emr("PutAutoTerminationPolicy", cluster() + ",\"AutoTerminationPolicy\":{\"IdleTimeout\":120}}")
.then().statusCode(200);
- call("GetAutoTerminationPolicy", cluster() + "}").then().statusCode(200)
+ emr("GetAutoTerminationPolicy", cluster() + "}").then().statusCode(200)
.body("AutoTerminationPolicy.IdleTimeout", equalTo(120));
- call("PutAutoTerminationPolicy", cluster() + ",\"AutoTerminationPolicy\":{\"IdleTimeout\":59}}")
+ emr("PutAutoTerminationPolicy", cluster() + ",\"AutoTerminationPolicy\":{\"IdleTimeout\":59}}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("PutAutoTerminationPolicy", cluster() + ",\"AutoTerminationPolicy\":{\"IdleTimeout\":604801}}")
+ emr("PutAutoTerminationPolicy", cluster() + ",\"AutoTerminationPolicy\":{\"IdleTimeout\":604801}}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("RemoveAutoTerminationPolicy", cluster() + "}").then().statusCode(200);
- call("GetAutoTerminationPolicy", cluster() + "}").then().statusCode(200)
+ emr("RemoveAutoTerminationPolicy", cluster() + "}").then().statusCode(200);
+ emr("GetAutoTerminationPolicy", cluster() + "}").then().statusCode(200)
.body("$", anEmptyMap());
}
@Test
@Order(5)
void autoScalingPolicyPutAndRemove() {
- call("PutAutoScalingPolicy", cluster() + ",\"InstanceGroupId\":\"" + coreGroupId + "\","
+ emr("PutAutoScalingPolicy", cluster() + ",\"InstanceGroupId\":\"" + coreGroupId + "\","
+ "\"AutoScalingPolicy\":{\"Constraints\":{\"MinCapacity\":2,\"MaxCapacity\":8},\"Rules\":"
+ "[{\"Name\":\"r\",\"Action\":{\"SimpleScalingPolicyConfiguration\":{\"ScalingAdjustment\":2}},"
+ "\"Trigger\":{\"CloudWatchAlarmDefinition\":{\"ComparisonOperator\":\"GREATER_THAN\","
@@ -142,22 +135,22 @@ class EmrPoliciesIntegrationTest {
.body("AutoScalingPolicy.Status.State", equalTo("ATTACHED"))
.body("AutoScalingPolicy.Constraints.MaxCapacity", equalTo(8))
.body("AutoScalingPolicy.Rules", hasSize(1));
- call("RemoveAutoScalingPolicy", cluster() + ",\"InstanceGroupId\":\"" + coreGroupId + "\"}")
+ emr("RemoveAutoScalingPolicy", cluster() + ",\"InstanceGroupId\":\"" + coreGroupId + "\"}")
.then().statusCode(200);
- call("ListInstanceGroups", cluster() + "}").then().statusCode(200)
+ emr("ListInstanceGroups", cluster() + "}").then().statusCode(200)
.body("InstanceGroups.find { it.Id == '" + coreGroupId + "' }.AutoScalingPolicy", nullValue());
}
@Test
@Order(6)
void autoScalingPolicyOnlyOnCoreOrTaskGroups() {
- call("PutAutoScalingPolicy", cluster() + ",\"InstanceGroupId\":\"" + masterGroupId + "\","
+ emr("PutAutoScalingPolicy", cluster() + ",\"InstanceGroupId\":\"" + masterGroupId + "\","
+ "\"AutoScalingPolicy\":" + AUTO_SCALING + "}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("PutAutoScalingPolicy", cluster() + ",\"InstanceGroupId\":\"ig-MISSING\","
+ emr("PutAutoScalingPolicy", cluster() + ",\"InstanceGroupId\":\"ig-MISSING\","
+ "\"AutoScalingPolicy\":" + AUTO_SCALING + "}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("RunJobFlow", "{\"Name\":\"bad\",\"Instances\":{\"InstanceGroups\":[{\"InstanceRole\":\"MASTER\","
+ emr("RunJobFlow", "{\"Name\":\"bad\",\"Instances\":{\"InstanceGroups\":[{\"InstanceRole\":\"MASTER\","
+ "\"InstanceType\":\"m5.xlarge\",\"InstanceCount\":1,\"AutoScalingPolicy\":" + AUTO_SCALING + "}]}}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
}
@@ -165,26 +158,26 @@ class EmrPoliciesIntegrationTest {
@Test
@Order(7)
void blockPublicAccessDefaultsThenPut() {
- call("GetBlockPublicAccessConfiguration", "{}").then().statusCode(200)
+ emr("GetBlockPublicAccessConfiguration", "{}").then().statusCode(200)
.body("BlockPublicAccessConfiguration.BlockPublicSecurityGroupRules", equalTo(true))
.body("BlockPublicAccessConfiguration.PermittedPublicSecurityGroupRuleRanges[0].MinRange", equalTo(22))
.body("BlockPublicAccessConfigurationMetadata.CreatedByArn", notNullValue())
.body("BlockPublicAccessConfigurationMetadata.CreationDateTime", notNullValue());
- call("PutBlockPublicAccessConfiguration", "{\"BlockPublicAccessConfiguration\":"
+ emr("PutBlockPublicAccessConfiguration", "{\"BlockPublicAccessConfiguration\":"
+ "{\"BlockPublicSecurityGroupRules\":true,\"PermittedPublicSecurityGroupRuleRanges\":"
+ "[{\"MinRange\":22,\"MaxRange\":22},{\"MinRange\":100,\"MaxRange\":101}]}}")
.then().statusCode(200);
- call("GetBlockPublicAccessConfiguration", "{}").then().statusCode(200)
+ emr("GetBlockPublicAccessConfiguration", "{}").then().statusCode(200)
.body("BlockPublicAccessConfiguration.PermittedPublicSecurityGroupRuleRanges", hasSize(2));
- call("PutBlockPublicAccessConfiguration", "{\"BlockPublicAccessConfiguration\":"
+ emr("PutBlockPublicAccessConfiguration", "{\"BlockPublicAccessConfiguration\":"
+ "{\"BlockPublicSecurityGroupRules\":true,\"PermittedPublicSecurityGroupRuleRanges\":"
+ "[{\"MinRange\":70000}]}}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("PutBlockPublicAccessConfiguration", "{\"BlockPublicAccessConfiguration\":"
+ emr("PutBlockPublicAccessConfiguration", "{\"BlockPublicAccessConfiguration\":"
+ "{\"BlockPublicSecurityGroupRules\":true,\"PermittedPublicSecurityGroupRuleRanges\":"
+ "[{\"MinRange\":22,\"MaxRange\":\"invalid\"}]}}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("PutBlockPublicAccessConfiguration", "{\"BlockPublicAccessConfiguration\":{}}")
+ emr("PutBlockPublicAccessConfiguration", "{\"BlockPublicAccessConfiguration\":{}}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emr/EmrReleaseCatalogCustomDefaultIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emr/EmrReleaseCatalogCustomDefaultIntegrationTest.java
new file mode 100644
index 0000000000..1375560c50
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/emr/EmrReleaseCatalogCustomDefaultIntegrationTest.java
@@ -0,0 +1,45 @@
+package io.github.hectorvent.floci.services.emr;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static io.github.hectorvent.floci.testing.EmrRequests.emr;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
+/**
+ * A default release label the catalogue does not describe is still a known release: it is listed
+ * (newest first here) and described with no applications, so Floci never calls its own default
+ * unknown.
+ */
+@QuarkusTest
+@TestProfile(EmrReleaseCatalogCustomDefaultIntegrationTest.FutureDefault.class)
+class EmrReleaseCatalogCustomDefaultIntegrationTest {
+
+ public static class FutureDefault implements QuarkusTestProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return Map.of("floci.services.emr.default-release-label", "emr-7.9.0");
+ }
+ }
+
+ @BeforeAll
+ static void configure() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void theConfiguredDefaultIsListedAndDescribed() {
+ emr("ListReleaseLabels", "{}").then().statusCode(200)
+ .body("ReleaseLabels[0]", equalTo("emr-7.9.0"));
+ emr("DescribeReleaseLabel", "{\"ReleaseLabel\":\"emr-7.9.0\"}").then().statusCode(200)
+ .body("ReleaseLabel", equalTo("emr-7.9.0"))
+ .body("Applications", hasSize(0));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emr/EmrReleaseCatalogIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emr/EmrReleaseCatalogIntegrationTest.java
new file mode 100644
index 0000000000..22c867cdc0
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/emr/EmrReleaseCatalogIntegrationTest.java
@@ -0,0 +1,109 @@
+package io.github.hectorvent.floci.services.emr;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import static io.github.hectorvent.floci.testing.EmrRequests.emr;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.everyItem;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.startsWith;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * The EMR release catalogue over the JSON protocol: ListReleaseLabels with its filters and paging,
+ * DescribeReleaseLabel, ListSupportedInstanceTypes, and InvalidRequestException for bad input.
+ */
+@QuarkusTest
+class EmrReleaseCatalogIntegrationTest {
+
+ @BeforeAll
+ static void configure() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void listReleaseLabelsNewestFirstWithFiltersAndPaging() {
+ emr("ListReleaseLabels", "{}").then().statusCode(200)
+ .body("ReleaseLabels", contains("emr-7.5.0", "emr-7.4.0", "emr-6.15.0", "emr-6.14.0", "emr-5.36.2"))
+ .body("NextToken", nullValue());
+ emr("ListReleaseLabels", "{\"Filters\":{\"Prefix\":\"emr-6\"}}").then().statusCode(200)
+ .body("ReleaseLabels", contains("emr-6.15.0", "emr-6.14.0"));
+ emr("ListReleaseLabels", "{\"Filters\":{\"Application\":\"[email protected]\"}}").then().statusCode(200)
+ .body("ReleaseLabels", contains("emr-6.15.0", "emr-6.14.0"));
+ // The version matches without regard to case too: the catalogue holds 4.14.3-HBase-1.4.
+ emr("ListReleaseLabels", "{\"Filters\":{\"Application\":\"[email protected]\"}}").then().statusCode(200)
+ .body("ReleaseLabels", contains("emr-5.36.2"));
+ emr("ListReleaseLabels", "{\"Filters\":{\"Application\":\"delta\"}}").then().statusCode(200)
+ .body("ReleaseLabels", everyItem(startsWith("emr-")))
+ .body("ReleaseLabels", hasSize(4));
+
+ String token = emr("ListReleaseLabels", "{\"MaxResults\":2}").then().statusCode(200)
+ .body("ReleaseLabels", contains("emr-7.5.0", "emr-7.4.0"))
+ .extract().path("NextToken");
+ emr("ListReleaseLabels", "{\"MaxResults\":2,\"NextToken\":\"" + token + "\"}").then().statusCode(200)
+ .body("ReleaseLabels", contains("emr-6.15.0", "emr-6.14.0"));
+
+ // A token belongs to the filters it was issued for.
+ String filtered = emr("ListReleaseLabels", "{\"MaxResults\":1,\"Filters\":{\"Prefix\":\"emr-6\"}}")
+ .then().statusCode(200).body("ReleaseLabels", contains("emr-6.15.0"))
+ .extract().path("NextToken");
+ emr("ListReleaseLabels", "{\"MaxResults\":1,\"Filters\":{\"Prefix\":\"emr-6\"},\"NextToken\":\""
+ + filtered + "\"}").then().statusCode(200)
+ .body("ReleaseLabels", contains("emr-6.14.0"))
+ .body("NextToken", nullValue());
+ emr("ListReleaseLabels", "{\"MaxResults\":1,\"Filters\":{\"Prefix\":\"emr-7\"},\"NextToken\":\""
+ + filtered + "\"}").then().statusCode(400)
+ .body("__type", equalTo("InvalidRequestException"));
+ }
+
+ @Test
+ void describeReleaseLabelReportsApplicationsAndOsReleases() {
+ emr("DescribeReleaseLabel", "{\"ReleaseLabel\":\"emr-7.5.0\"}").then().statusCode(200)
+ .body("ReleaseLabel", equalTo("emr-7.5.0"))
+ .body("Applications.find { it.Name == 'spark' }.Version", equalTo("3.5.2"))
+ .body("Applications.find { it.Name == 'hadoop' }.Version", equalTo("3.4.0"))
+ .body("AvailableOSReleases.Label", hasItem(startsWith("2023.")));
+ // No label: the latest release.
+ emr("DescribeReleaseLabel", "{}").then().statusCode(200)
+ .body("ReleaseLabel", equalTo("emr-7.5.0"));
+ // MaxResults is reserved for future use: every application comes back, with no token.
+ int all = emr("DescribeReleaseLabel", "{\"ReleaseLabel\":\"emr-5.36.2\"}").then().statusCode(200)
+ .extract().path("Applications.size()");
+ emr("DescribeReleaseLabel", "{\"ReleaseLabel\":\"emr-5.36.2\",\"MaxResults\":5}").then().statusCode(200)
+ .body("Applications", hasSize(all))
+ .body("NextToken", nullValue());
+ assertTrue(all > 5);
+ }
+
+ @Test
+ void listSupportedInstanceTypesForARelease() {
+ emr("ListSupportedInstanceTypes", "{\"ReleaseLabel\":\"emr-7.5.0\"}").then().statusCode(200)
+ .body("SupportedInstanceTypes.find { it.Type == 'm5.xlarge' }.VCPU", equalTo(4))
+ .body("SupportedInstanceTypes.find { it.Type == 'm5.xlarge' }.EbsStorageOnly", equalTo(true))
+ .body("SupportedInstanceTypes.find { it.Type == 'm6g.xlarge' }.Architecture", equalTo("AARCH64"))
+ .body("SupportedInstanceTypes.find { it.Type == 'm5.xlarge' }.Architecture", equalTo("X86_64"))
+ .body("SupportedInstanceTypes.find { it.Type == 'm5d.xlarge' }.StorageGB", equalTo(150));
+ }
+
+ @Test
+ void badInputIsAnInvalidRequest() {
+ for (String[] bad : new String[][] {
+ {"DescribeReleaseLabel", "{\"ReleaseLabel\":\"emr-1.0.0\"}"},
+ {"ListSupportedInstanceTypes", "{\"ReleaseLabel\":\"emr-1.0.0\"}"},
+ {"ListSupportedInstanceTypes", "{}"},
+ {"ListReleaseLabels", "{\"MaxResults\":0}"},
+ {"ListReleaseLabels", "{\"MaxResults\":101}"},
+ {"ListReleaseLabels", "{\"NextToken\":\"next\"}"},
+ {"ListReleaseLabels", "{\"NextToken\":\"-1\"}"},
+ {"ListSupportedInstanceTypes", "{\"ReleaseLabel\":\"emr-7.5.0\",\"Marker\":\"2\"}"}}) {
+ emr(bad[0], bad[1]).then().statusCode(400)
+ .body("__type", equalTo("InvalidRequestException"));
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emr/EmrResizeIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emr/EmrResizeIntegrationTest.java
index e583dc770e..190c732561 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emr/EmrResizeIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emr/EmrResizeIntegrationTest.java
@@ -11,7 +11,7 @@ import org.junit.jupiter.api.TestMethodOrder;
import java.util.List;
-import static io.restassured.RestAssured.given;
+import static io.github.hectorvent.floci.testing.EmrRequests.emr;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.nullValue;
@@ -24,9 +24,6 @@ import static org.hamcrest.Matchers.nullValue;
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
class EmrResizeIntegrationTest {
- private static final String CT = "application/x-amz-json-1.1";
- private static final String PREFIX = "ElasticMapReduce.";
-
private static String groupsClusterId;
private static String masterGroupId;
private static String coreGroupId;
@@ -40,20 +37,15 @@ class EmrResizeIntegrationTest {
RestAssuredJsonUtils.configureAwsContentTypes();
}
- private static Response call(String action, String body) {
- return given().contentType(CT).header("X-Amz-Target", PREFIX + action)
- .body(body).when().post("/");
- }
-
private static String groupId(String type) {
- return call("ListInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\"}").jsonPath()
+ return emr("ListInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\"}").jsonPath()
.getString("InstanceGroups.find { it.InstanceGroupType == '" + type + "' }.Id");
}
@Test
@Order(1)
void runJobFlowStoresBootstrapActions() {
- Response resp = call("RunJobFlow",
+ Response resp = emr("RunJobFlow",
"{\"Name\":\"resize\",\"ReleaseLabel\":\"emr-7.5.0\","
+ "\"BootstrapActions\":[{\"Name\":\"install\",\"ScriptBootstrapAction\":"
+ "{\"Path\":\"s3://bucket/install.sh\",\"Args\":[\"--fast\",\"1\"]}},"
@@ -69,11 +61,11 @@ class EmrResizeIntegrationTest {
coreGroupId = groupId("CORE");
taskGroupId = groupId("TASK");
- call("ListInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\"}").then().statusCode(200)
+ emr("ListInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\"}").then().statusCode(200)
.body("InstanceGroups.find { it.Id == '" + taskGroupId + "' }.Configurations", hasSize(0))
.body("InstanceGroups.find { it.Id == '" + coreGroupId + "' }.Configurations", nullValue());
- call("ListBootstrapActions", "{\"ClusterId\":\"" + groupsClusterId + "\"}").then().statusCode(200)
+ emr("ListBootstrapActions", "{\"ClusterId\":\"" + groupsClusterId + "\"}").then().statusCode(200)
.body("BootstrapActions", hasSize(2))
.body("BootstrapActions[0].Name", equalTo("install"))
.body("BootstrapActions[0].ScriptPath", equalTo("s3://bucket/install.sh"))
@@ -84,25 +76,25 @@ class EmrResizeIntegrationTest {
@Test
@Order(2)
void listBootstrapActionsEmptyAndValidated() {
- Response resp = call("RunJobFlow", "{\"Name\":\"plain\",\"Instances\":{\"InstanceGroups\":"
+ Response resp = emr("RunJobFlow", "{\"Name\":\"plain\",\"Instances\":{\"InstanceGroups\":"
+ "[{\"InstanceRole\":\"MASTER\",\"InstanceType\":\"m5.xlarge\",\"InstanceCount\":1}]}}");
- call("ListBootstrapActions", "{\"ClusterId\":\"" + resp.jsonPath().getString("JobFlowId") + "\"}")
+ emr("ListBootstrapActions", "{\"ClusterId\":\"" + resp.jsonPath().getString("JobFlowId") + "\"}")
.then().statusCode(200).body("BootstrapActions", hasSize(0));
- call("ListBootstrapActions", "{\"ClusterId\":\"j-MISSING\"}")
+ emr("ListBootstrapActions", "{\"ClusterId\":\"j-MISSING\"}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("RunJobFlow", "{\"Name\":\"bad\",\"BootstrapActions\":[{\"Name\":\"x\"}]}")
+ emr("RunJobFlow", "{\"Name\":\"bad\",\"BootstrapActions\":[{\"Name\":\"x\"}]}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
}
@Test
@Order(3)
void modifyInstanceGroupsResizesAndReconfigures() {
- call("ModifyInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\",\"InstanceGroups\":["
+ emr("ModifyInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\",\"InstanceGroups\":["
+ "{\"InstanceGroupId\":\"" + coreGroupId + "\",\"InstanceCount\":4},"
+ "{\"InstanceGroupId\":\"" + taskGroupId + "\",\"InstanceCount\":0,\"Configurations\":"
+ "[{\"Classification\":\"yarn-site\",\"Properties\":{\"yarn.nodemanager.vmem-check-enabled\":\"false\"}}]}]}")
.then().statusCode(200);
- call("ListInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\"}").then().statusCode(200)
+ emr("ListInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\"}").then().statusCode(200)
.body("InstanceGroups.find { it.Id == '" + coreGroupId + "' }.RequestedInstanceCount", equalTo(4))
.body("InstanceGroups.find { it.Id == '" + coreGroupId + "' }.RunningInstanceCount", equalTo(4))
.body("InstanceGroups.find { it.Id == '" + coreGroupId + "' }.Configurations", nullValue())
@@ -113,16 +105,16 @@ class EmrResizeIntegrationTest {
.body("InstanceGroups.find { it.Id == '" + taskGroupId + "' }.LastSuccessfullyAppliedConfigurationsVersion",
equalTo(1));
// 1 master + 4 core + 0 task.
- call("ListInstances", "{\"ClusterId\":\"" + groupsClusterId + "\"}").then().statusCode(200)
+ emr("ListInstances", "{\"ClusterId\":\"" + groupsClusterId + "\"}").then().statusCode(200)
.body("Instances", hasSize(5));
}
@Test
@Order(4)
void modifyInstanceGroupsWithoutClusterIdFindsTheGroup() {
- call("ModifyInstanceGroups", "{\"InstanceGroups\":[{\"InstanceGroupId\":\"" + coreGroupId
+ emr("ModifyInstanceGroups", "{\"InstanceGroups\":[{\"InstanceGroupId\":\"" + coreGroupId
+ "\",\"InstanceCount\":3}]}").then().statusCode(200);
- call("ListInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\"}").then().statusCode(200)
+ emr("ListInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\"}").then().statusCode(200)
.body("InstanceGroups.find { it.Id == '" + coreGroupId + "' }.RunningInstanceCount", equalTo(3));
}
@@ -130,40 +122,40 @@ class EmrResizeIntegrationTest {
@Order(5)
void modifyInstanceGroupsIsValidatedBeforeAnyChange() {
// The second modification is rejected, so the first is not applied either.
- call("ModifyInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\",\"InstanceGroups\":["
+ emr("ModifyInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\",\"InstanceGroups\":["
+ "{\"InstanceGroupId\":\"" + coreGroupId + "\",\"InstanceCount\":9},"
+ "{\"InstanceGroupId\":\"" + masterGroupId + "\",\"InstanceCount\":3}]}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("ListInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\"}").then().statusCode(200)
+ emr("ListInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\"}").then().statusCode(200)
.body("InstanceGroups.find { it.Id == '" + coreGroupId + "' }.RunningInstanceCount", equalTo(3));
- call("ModifyInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\",\"InstanceGroups\":["
+ emr("ModifyInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\",\"InstanceGroups\":["
+ "{\"InstanceGroupId\":\"" + coreGroupId + "\",\"InstanceCount\":-1}]}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("ModifyInstanceGroups", "{\"InstanceGroups\":[{\"InstanceGroupId\":\"ig-MISSING\",\"InstanceCount\":1}]}")
+ emr("ModifyInstanceGroups", "{\"InstanceGroups\":[{\"InstanceGroupId\":\"ig-MISSING\",\"InstanceCount\":1}]}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("ModifyInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\",\"InstanceGroups\":[]}")
+ emr("ModifyInstanceGroups", "{\"ClusterId\":\"" + groupsClusterId + "\",\"InstanceGroups\":[]}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
}
@Test
@Order(6)
void modifyInstanceFleetResizes() {
- Response resp = call("RunJobFlow", "{\"Name\":\"fleets\",\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true,"
+ Response resp = emr("RunJobFlow", "{\"Name\":\"fleets\",\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true,"
+ "\"InstanceFleets\":[{\"InstanceFleetType\":\"MASTER\",\"TargetOnDemandCapacity\":1,"
+ "\"InstanceTypeConfigs\":[{\"InstanceType\":\"m5.xlarge\"}]},"
+ "{\"InstanceFleetType\":\"CORE\",\"TargetOnDemandCapacity\":2,\"TargetSpotCapacity\":2,"
+ "\"InstanceTypeConfigs\":[{\"InstanceType\":\"m5.xlarge\"}]}]}}");
resp.then().statusCode(200);
fleetClusterId = resp.jsonPath().getString("JobFlowId");
- Response fleets = call("ListInstanceFleets", "{\"ClusterId\":\"" + fleetClusterId + "\"}");
+ Response fleets = emr("ListInstanceFleets", "{\"ClusterId\":\"" + fleetClusterId + "\"}");
masterFleetId = fleets.jsonPath().getString("InstanceFleets.find { it.InstanceFleetType == 'MASTER' }.Id");
coreFleetId = fleets.jsonPath().getString("InstanceFleets.find { it.InstanceFleetType == 'CORE' }.Id");
- call("ModifyInstanceFleet", "{\"ClusterId\":\"" + fleetClusterId + "\",\"InstanceFleet\":"
+ emr("ModifyInstanceFleet", "{\"ClusterId\":\"" + fleetClusterId + "\",\"InstanceFleet\":"
+ "{\"InstanceFleetId\":\"" + coreFleetId + "\",\"TargetSpotCapacity\":6}}")
.then().statusCode(200);
- call("ListInstanceFleets", "{\"ClusterId\":\"" + fleetClusterId + "\"}").then().statusCode(200)
+ emr("ListInstanceFleets", "{\"ClusterId\":\"" + fleetClusterId + "\"}").then().statusCode(200)
.body("InstanceFleets.find { it.Id == '" + coreFleetId + "' }.TargetOnDemandCapacity", equalTo(2))
.body("InstanceFleets.find { it.Id == '" + coreFleetId + "' }.TargetSpotCapacity", equalTo(6))
.body("InstanceFleets.find { it.Id == '" + coreFleetId + "' }.ProvisionedSpotCapacity", equalTo(6));
@@ -172,16 +164,16 @@ class EmrResizeIntegrationTest {
@Test
@Order(7)
void modifyInstanceFleetIsValidated() {
- call("ModifyInstanceFleet", "{\"ClusterId\":\"" + fleetClusterId + "\",\"InstanceFleet\":"
+ emr("ModifyInstanceFleet", "{\"ClusterId\":\"" + fleetClusterId + "\",\"InstanceFleet\":"
+ "{\"InstanceFleetId\":\"" + masterFleetId + "\",\"TargetOnDemandCapacity\":3}}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("ModifyInstanceFleet", "{\"ClusterId\":\"" + fleetClusterId + "\",\"InstanceFleet\":"
+ emr("ModifyInstanceFleet", "{\"ClusterId\":\"" + fleetClusterId + "\",\"InstanceFleet\":"
+ "{\"InstanceFleetId\":\"" + coreFleetId + "\",\"TargetOnDemandCapacity\":-1}}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("ModifyInstanceFleet", "{\"ClusterId\":\"" + fleetClusterId + "\",\"InstanceFleet\":"
+ emr("ModifyInstanceFleet", "{\"ClusterId\":\"" + fleetClusterId + "\",\"InstanceFleet\":"
+ "{\"InstanceFleetId\":\"if-MISSING\",\"TargetOnDemandCapacity\":1}}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
- call("ModifyInstanceFleet", "{\"ClusterId\":\"j-MISSING\",\"InstanceFleet\":"
+ emr("ModifyInstanceFleet", "{\"ClusterId\":\"j-MISSING\",\"InstanceFleet\":"
+ "{\"InstanceFleetId\":\"" + coreFleetId + "\",\"TargetOnDemandCapacity\":1}}")
.then().statusCode(400).body("__type", equalTo("InvalidRequestException"));
}
diff --git a/src/test/java/io/github/hectorvent/floci/testing/EmrRequests.java b/src/test/java/io/github/hectorvent/floci/testing/EmrRequests.java
new file mode 100644
index 0000000000..f16f262530
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/testing/EmrRequests.java
@@ -0,0 +1,24 @@
+package io.github.hectorvent.floci.testing;
+
+import io.restassured.response.Response;
+
+import static io.restassured.RestAssured.given;
+
+/** One EMR JSON-protocol call, in the default Region or a named one. */
+public final class EmrRequests {
+ private EmrRequests() {}
+
+ public static Response emr(String action, String body) {
+ return given().contentType("application/x-amz-json-1.1")
+ .header("X-Amz-Target", "ElasticMapReduce." + action)
+ .body(body).when().post("/");
+ }
+
+ public static Response emr(String region, String action, String body) {
+ return given().contentType("application/x-amz-json-1.1")
+ .header("Authorization", "AWS4-HMAC-SHA256 Credential=AKID/20260215/" + region
+ + "/elasticmapreduce/aws4_request, SignedHeaders=host, Signature=abc")
+ .header("X-Amz-Target", "ElasticMapReduce." + action)
+ .body(body).when().post("/");
+ }
+}