Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(emr): use the cluster's regional private DNS domain for instances and the master (#5169)

Hector Ventura4115becdf2be
diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java b/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java
index d38e2776ac..d18c8d9c68 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emr/EmrHandler.java
@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.common.AwsEndpoints;
import io.github.hectorvent.floci.core.common.AwsErrorResponse;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.AwsJson11Controller;
+import io.github.hectorvent.floci.core.common.AwsRegions;
import io.github.hectorvent.floci.services.emr.model.EmrBlockPublicAccess;
import io.github.hectorvent.floci.services.emr.model.EmrCluster;
import io.github.hectorvent.floci.services.emr.model.EmrInstanceFleet;
@@ -492,7 +493,7 @@ public class EmrHandler {
}
node.put("NormalizedInstanceHours", c.getNormalizedInstanceHours());
if (c.getMasterPublicDnsName() != null) {
- node.put("MasterPublicDnsName", c.getMasterPublicDnsName());
+ node.put("MasterPublicDnsName", EmrService.masterDnsName(c.getRegion()));
}
if (c.getAutoScalingRole() != null) {
node.put("AutoScalingRole", c.getAutoScalingRole());
@@ -628,8 +629,9 @@ public class EmrHandler {
node.put("Id", "ci-" + index + cluster.getId());
node.put("Ec2InstanceId", "i-" + String.format("%017d", index));
node.put("PublicDnsName", AwsEndpoints.ec2PublicDns("203.0.113." + index, cluster.getRegion()));
- node.put("PrivateDnsName", "ip-10-0-0-" + index + ".ec2.internal");
- node.put("PrivateIpAddress", "10.0.0." + index);
+ String privateIp = "10.0.0." + index;
+ node.put("PrivateDnsName", AwsRegions.ec2PrivateIpDnsName(privateIp, cluster.getRegion()));
+ node.put("PrivateIpAddress", privateIp);
node.put("InstanceGroupId", group.getId());
if (group.getMarket() != null) {
node.put("Market", group.getMarket());
diff --git a/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java b/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java
index 42c065e871..a678edf05e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/emr/EmrService.java
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.AwsRegions;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
@@ -86,7 +87,7 @@ public class EmrService {
cluster.setInstanceCollectionType(cluster.getInstanceFleets().isEmpty()
? "INSTANCE_GROUP" : "INSTANCE_FLEET");
cluster.setAutoTerminate(!cluster.isKeepJobFlowAliveWhenNoSteps());
- cluster.setMasterPublicDnsName("ip-10-0-0-1." + region + ".compute.internal");
+ cluster.setMasterPublicDnsName(masterDnsName(region));
cluster.setCreationDateTime(Instant.now());
for (EmrInstanceGroup g : cluster.getInstanceGroups()) {
g.setId("ig-" + randomId(13));
@@ -108,6 +109,11 @@ public class EmrService {
return cluster;
}
+ /** The master node's private DNS name, derived from the region so a stored name never goes stale. */
+ static String masterDnsName(String region) {
+ return AwsRegions.ec2PrivateIpDnsName("10.0.0.1", region);
+ }
+
public EmrCluster describeCluster(String id) {
return requireCluster(id);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emr/EmrClusterRegionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emr/EmrClusterRegionIntegrationTest.java
index b71fb5ed73..db0f03bb5f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emr/EmrClusterRegionIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emr/EmrClusterRegionIntegrationTest.java
@@ -70,4 +70,27 @@ class EmrClusterRegionIntegrationTest {
call(HOME, "TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}")
.then().statusCode(200);
}
+
+ // Catches: an instance's or the master's private DNS name in the wrong region's domain.
+ @Test
+ void listedInstancesUseTheClustersRegionalPrivateDnsDomain() {
+ String clusterId = call(HOME, "RunJobFlow",
+ "{\"Name\":\"region-dns\",\"ReleaseLabel\":\"emr-7.5.0\","
+ + "\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true,"
+ + "\"InstanceGroups\":[{\"Name\":\"master\",\"InstanceRole\":\"MASTER\","
+ + "\"InstanceType\":\"m5.xlarge\",\"InstanceCount\":1}]}}")
+ .then().statusCode(200)
+ .extract().path("JobFlowId");
+
+ call(HOME, "ListInstances", "{\"ClusterId\":\"" + clusterId + "\"}")
+ .then().statusCode(200)
+ .body("Instances[0].PrivateDnsName", equalTo("ip-10-0-0-1.eu-west-1.compute.internal"))
+ .body("Instances[0].PrivateIpAddress", equalTo("10.0.0.1"));
+ call(HOME, "DescribeCluster", "{\"ClusterId\":\"" + clusterId + "\"}")
+ .then().statusCode(200)
+ .body("Cluster.MasterPublicDnsName", equalTo("ip-10-0-0-1.eu-west-1.compute.internal"));
+
+ call(HOME, "TerminateJobFlows", "{\"JobFlowIds\":[\"" + clusterId + "\"]}")
+ .then().statusCode(200);
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java
index c684d29ce6..91357b16b6 100644
--- a/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/emr/EmrIntegrationTest.java
@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.emr;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.restassured.response.Response;
import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
@@ -10,6 +11,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.notNullValue;
@@ -30,6 +32,9 @@ class EmrIntegrationTest {
private static String clusterId;
private static String stepId;
+ @Inject
+ EmrService emrService;
+
@BeforeAll
static void configure() {
RestAssuredJsonUtils.configureAwsContentTypes();
@@ -67,7 +72,8 @@ class EmrIntegrationTest {
.body("Cluster.Status.State", equalTo("WAITING"))
.body("Cluster.ReleaseLabel", equalTo("emr-7.5.0"))
.body("Cluster.InstanceCollectionType", equalTo("INSTANCE_GROUP"))
- .body("Cluster.AutoTerminate", equalTo(false));
+ .body("Cluster.AutoTerminate", equalTo(false))
+ .body("Cluster.MasterPublicDnsName", equalTo("ip-10-0-0-1.ec2.internal"));
}
@Test
@@ -93,7 +99,8 @@ class EmrIntegrationTest {
call("ListInstances", "{\"ClusterId\":\"" + clusterId + "\"}")
.then().statusCode(200)
.body("Instances", hasSize(3)) // 1 master + 2 core
- .body("Instances[0].Status.State", equalTo("RUNNING"));
+ .body("Instances[0].Status.State", equalTo("RUNNING"))
+ .body("Instances[0].PrivateDnsName", endsWith(".ec2.internal"));
}
@Test
@@ -245,4 +252,24 @@ class EmrIntegrationTest {
.then().statusCode(200)
.body("Cluster.Status.State", equalTo("WAITING"));
}
+
+ // Catches: a cluster stored before the DNS fix keeps reporting its old us-east-1 master name.
+ @Test
+ void storedClusterReportsTheRegionalMasterDnsName() {
+ String id = call("RunJobFlow",
+ "{\"Name\":\"stale-dns\",\"ReleaseLabel\":\"emr-7.5.0\","
+ + "\"Instances\":{\"KeepJobFlowAliveWhenNoSteps\":true,"
+ + "\"InstanceGroups\":[{\"Name\":\"master\",\"InstanceRole\":\"MASTER\","
+ + "\"InstanceType\":\"m5.xlarge\",\"InstanceCount\":1}]}}")
+ .then().statusCode(200)
+ .extract().path("JobFlowId");
+ emrService.describeCluster(id).setMasterPublicDnsName("ip-10-0-0-1.us-east-1.compute.internal");
+
+ call("DescribeCluster", "{\"ClusterId\":\"" + id + "\"}")
+ .then().statusCode(200)
+ .body("Cluster.MasterPublicDnsName", equalTo("ip-10-0-0-1.ec2.internal"));
+
+ call("TerminateJobFlows", "{\"JobFlowIds\":[\"" + id + "\"]}")
+ .then().statusCode(200);
+ }
}