Light, fluffy, and always free - The AWS Local Emulator alternative
fix(emr): use the cluster's regional private DNS domain for instances and the master (#5169)
Hector Ventura4115becdf2be
diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java b/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.javaindex d38e2776ac..d18c8d9c68 100644--- a/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.common.AwsEndpoints;import io.github.hectorvent.floci.core.common.AwsErrorResponse;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.AwsJson11Controller;+import io.github.hectorvent.floci.core.common.AwsRegions;import io.github.hectorvent.floci.services.emr.model.EmrBlockPublicAccess;import io.github.hectorvent.floci.services.emr.model.EmrCluster;import io.github.hectorvent.floci.services.emr.model.EmrInstanceFleet;@@ -492,7 +493,7 @@ public class EmrHandler {}node.put("NormalizedInstanceHours", c.getNormalizedInstanceHours());if (c.getMasterPublicDnsName() != null) {- node.put("MasterPublicDnsName", c.getMasterPublicDnsName());+ node.put("MasterPublicDnsName", EmrService.masterDnsName(c.getRegion()));}if (c.getAutoScalingRole() != null) {node.put("AutoScalingRole", c.getAutoScalingRole());@@ -628,8 +629,9 @@ public class EmrHandler {node.put("Id", "ci-" + index + cluster.getId());node.put("Ec2InstanceId", "i-" + String.format("%017d", index));node.put("PublicDnsName", AwsEndpoints.ec2PublicDns("203.0.113." + index, cluster.getRegion()));- node.put("PrivateDnsName", "ip-10-0-0-" + index + ".ec2.internal");- node.put("PrivateIpAddress", "10.0.0." + index);+ String privateIp = "10.0.0." + index;+ node.put("PrivateDnsName", AwsRegions.ec2PrivateIpDnsName(privateIp, cluster.getRegion()));+ node.put("PrivateIpAddress", privateIp);node.put("InstanceGroupId", group.getId());if (group.getMarket() != null) {node.put("Market", group.getMarket());diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java b/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.javaindex 42c065e871..a678edf05e 100644--- a/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java+++ b/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java@@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference;import com.fasterxml.jackson.databind.JsonNode;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.AwsRegions;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;@@ -86,7 +87,7 @@ public class EmrService {cluster.setInstanceCollectionType(cluster.getInstanceFleets().isEmpty()? "INSTANCE_GROUP" : "INSTANCE_FLEET");cluster.setAutoTerminate(!cluster.isKeepJobFlowAliveWhenNoSteps());- cluster.setMasterPublicDnsName("ip-10-0-0-1." + region + ".compute.internal");+ cluster.setMasterPublicDnsName(masterDnsName(region));cluster.setCreationDateTime(Instant.now());for (EmrInstanceGroup g : cluster.getInstanceGroups()) {g.setId("ig-" + randomId(13));@@ -108,6 +109,11 @@ public class EmrService {return cluster;}+ /** The master node's private DNS name, derived from the region so a stored name never goes stale. */+ static String masterDnsName(String region) {+ return AwsRegions.ec2PrivateIpDnsName("10.0.0.1", region);+ }+public EmrCluster describeCluster(String id) {return requireCluster(id);}diff --git a/src/test/java/io/github/hectorvent/floci/services/emr/EmrClusterRegionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emr/EmrClusterRegionIntegrationTest.javaindex b71fb5ed73..db0f03bb5f 100644--- a/src/test/java/io/github/hectorvent/floci/services/emr/EmrClusterRegionIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/emr/EmrClusterRegionIntegrationTest.java@@ -70,4 +70,27 @@ class EmrClusterRegionIntegrationTest {call(HOME, "TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}").then().statusCode(200);}++ // Catches: an instance's or the master's private DNS name in the wrong region's domain.+ @Test+ void listedInstancesUseTheClustersRegionalPrivateDnsDomain() {+ String clusterId = call(HOME, "RunJobFlow",+ "{\"Name\":\"region-dns\",\"ReleaseLabel\":\"emr-7.5.0\","+ + "\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true,"+ + "\"InstanceGroups\":[{\"Name\":\"master\",\"InstanceRole\":\"MASTER\","+ + "\"InstanceType\":\"m5.xlarge\",\"InstanceCount\":1}]}}")+ .then().statusCode(200)+ .extract().path("JobFlowId");++ call(HOME, "ListInstances", "{\"ClusterId\":\"" + clusterId + "\"}")+ .then().statusCode(200)+ .body("Instances[0].PrivateDnsName", equalTo("ip-10-0-0-1.eu-west-1.compute.internal"))+ .body("Instances[0].PrivateIpAddress", equalTo("10.0.0.1"));+ call(HOME, "DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")+ .then().statusCode(200)+ .body("Cluster.MasterPublicDnsName", equalTo("ip-10-0-0-1.eu-west-1.compute.internal"));++ call(HOME, "TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}")+ .then().statusCode(200);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.javaindex c684d29ce6..91357b16b6 100644--- a/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.emr;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.restassured.response.Response;import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.MethodOrderer;import org.junit.jupiter.api.Order;@@ -10,6 +11,7 @@ import org.junit.jupiter.api.Test;import org.junit.jupiter.api.TestMethodOrder;import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.endsWith;import static org.hamcrest.Matchers.equalTo;import static org.hamcrest.Matchers.hasSize;import static org.hamcrest.Matchers.notNullValue;@@ -30,6 +32,9 @@ class EmrIntegrationTest {private static String clusterId;private static String stepId;+ @Inject+ EmrService emrService;+@BeforeAllstatic void configure() {RestAssuredJsonUtils.configureAwsContentTypes();@@ -67,7 +72,8 @@ class EmrIntegrationTest {.body("Cluster.Status.State", equalTo("WAITING")).body("Cluster.ReleaseLabel", equalTo("emr-7.5.0")).body("Cluster.InstanceCollectionType", equalTo("INSTANCE_GROUP"))- .body("Cluster.AutoTerminate", equalTo(false));+ .body("Cluster.AutoTerminate", equalTo(false))+ .body("Cluster.MasterPublicDnsName", equalTo("ip-10-0-0-1.ec2.internal"));}@Test@@ -93,7 +99,8 @@ class EmrIntegrationTest {call("ListInstances", "{\"ClusterId\":\"" + clusterId + "\"}").then().statusCode(200).body("Instances", hasSize(3)) // 1 master + 2 core- .body("Instances[0].Status.State", equalTo("RUNNING"));+ .body("Instances[0].Status.State", equalTo("RUNNING"))+ .body("Instances[0].PrivateDnsName", endsWith(".ec2.internal"));}@Test@@ -245,4 +252,24 @@ class EmrIntegrationTest {.then().statusCode(200).body("Cluster.Status.State", equalTo("WAITING"));}++ // Catches: a cluster stored before the DNS fix keeps reporting its old us-east-1 master name.+ @Test+ void storedClusterReportsTheRegionalMasterDnsName() {+ String id = call("RunJobFlow",+ "{\"Name\":\"stale-dns\",\"ReleaseLabel\":\"emr-7.5.0\","+ + "\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true,"+ + "\"InstanceGroups\":[{\"Name\":\"master\",\"InstanceRole\":\"MASTER\","+ + "\"InstanceType\":\"m5.xlarge\",\"InstanceCount\":1}]}}")+ .then().statusCode(200)+ .extract().path("JobFlowId");+ emrService.describeCluster(id).setMasterPublicDnsName("ip-10-0-0-1.us-east-1.compute.internal");++ call("DescribeCluster", "{\"ClusterId\":\"" + id + "\"}")+ .then().statusCode(200)+ .body("Cluster.MasterPublicDnsName", equalTo("ip-10-0-0-1.ec2.internal"));++ call("TerminateJobFlows", "{\"JobFlowIds\":[\"" + id + "\"]}")+ .then().statusCode(200);+ }}