Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dms): list the default replication subnet group (#5771)
Allen Sanborn23e1b3a97cb8
diff --git a/docs/services/dms.md b/docs/services/dms.mdindex d406c2cb63..c817223354 100644--- a/docs/services/dms.md+++ b/docs/services/dms.md@@ -49,6 +49,11 @@ those subnets actually have. No data is migrated: the resources are control-plan- `SubnetGroupStatus` is immediately `Complete`, every subnet reports `Active`,`SupportedNetworkTypes` is `["IPV4"]`, and `IsReadOnly` is always `false`: a read-onlygroup is one DMS manages for a zero-ETL integration, which Floci does not emulate.+- The `default` group is synthesised from the region's default-VPC subnets, and only+ exists once `CreateReplicationInstance` has put an instance in it (an instance created+ without `ReplicationSubnetGroupIdentifier`), as AWS creates it the first time it is needed.+ Until then it is neither listed nor found by the filter. Unlike AWS, it disappears again+ once no instance in the region uses it.- `DescribeReplicationSubnetGroups` supports the `replication-subnet-group-id` filter.A filter naming a group that does not exist returns `ResourceNotFoundFault`, which ishow Terraform detects a group deleted outside its state.diff --git a/src/main/java/io/github/hectorvent/floci/services/dms/DmsService.java b/src/main/java/io/github/hectorvent/floci/services/dms/DmsService.javaindex b2f0b9b5eb..831e9e8479 100644--- a/src/main/java/io/github/hectorvent/floci/services/dms/DmsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dms/DmsService.java@@ -249,12 +249,18 @@ public class DmsService implements Resettable {/*** "default" cannot be created, so it is never stored: it resolves to the region's default-VPC- * subnets at read time, which is the group an instance created without one lands in.+ * subnets at read time, which is the group an instance created without one lands in. AWS only+ * creates that group the first time an instance needs it, so it exists here only while a+ * replication instance in the region uses it. That check comes first, so a region with no such+ * instance never reaches EC2 and never seeds its default VPC.*/public Optional<ReplicationSubnetGroup> findReplicationSubnetGroup(String region, String identifier) {if (!"default".equals(identifier)) {return subnetGroups.get(storageKey(region, identifier));}+ if (!subnetGroupInUse(region, identifier)) {+ return Optional.empty();+ }List<String> defaultSubnets = ec2Service.describeSubnets(region, List.of(), Map.of()).stream().filter(Subnet::isDefaultForAz).map(Subnet::getSubnetId)@@ -363,7 +369,9 @@ public class DmsService implements Resettable {* Pages by identifier through the shared opaque-cursor helper, so {@code Marker} stays* resumable when a group is created or deleted between requests. DMS documents a default* {@code MaxRecords} of 100 and a valid range of 20 to 100, and rejects a value outside that- * range rather than clamping it.+ * range rather than clamping it. The synthetic {@code default} group is listed and filterable+ * like a stored one whenever {@link #findReplicationSubnetGroup} resolves it, which is only once+ * a replication instance in the region uses it.*/public PaginatedResult<ReplicationSubnetGroup> describeReplicationSubnetGroups(JsonNode request, String region) {Integer maxRecords = maxRecords(request);@@ -372,11 +380,12 @@ public class DmsService implements Resettable {List<ReplicationSubnetGroup> matching;if (!requestedIdentifiers.isEmpty()) {matching = requestedIdentifiers.stream()- .map(identifier -> subnetGroups.get(storageKey(region, identifier))+ .map(identifier -> findReplicationSubnetGroup(region, identifier).orElseThrow(() -> notFound(identifier))).toList();} else {- matching = subnetGroups.scan(key -> key.startsWith(region + "::"));+ matching = new ArrayList<>(subnetGroups.scan(key -> key.startsWith(region + "::")));+ findReplicationSubnetGroup(region, "default").ifPresent(matching::add);}return Pagination.paginate(matching, ReplicationSubnetGroup::getReplicationSubnetGroupIdentifier,maxRecords, marker, DEFAULT_MAX_RECORDS, MAXIMUM_MAX_RECORDS,@@ -389,16 +398,19 @@ public class DmsService implements Resettable {if (subnetGroups.get(key).isEmpty()) {throw notFound(identifier);}- boolean inUse = regionScan(instances, region).stream()- .anyMatch(instance -> identifier.equals(instance.getAttributes()- .path("ReplicationSubnetGroup").path("ReplicationSubnetGroupIdentifier").asText(null)));- if (inUse) {+ if (subnetGroupInUse(region, identifier)) {throw invalidState("Replication subnet group " + identifier+ " is in use by a replication instance and cannot be deleted.");}subnetGroups.delete(key);}+ private boolean subnetGroupInUse(String region, String identifier) {+ return regionScan(instances, region).stream()+ .anyMatch(instance -> identifier.equals(instance.getAttributes()+ .path("ReplicationSubnetGroup").path("ReplicationSubnetGroupIdentifier").asText(null)));+ }+public List<ResourceTag> listTagsForResource(JsonNode request, String region) {List<String> arnList = arnList(request);if (!arnList.isEmpty()) {diff --git a/src/test/java/io/github/hectorvent/floci/services/dms/DmsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dms/DmsIntegrationTest.javaindex 66d2ff75de..0b436c38da 100644--- a/src/test/java/io/github/hectorvent/floci/services/dms/DmsIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dms/DmsIntegrationTest.java@@ -144,6 +144,67 @@ class DmsIntegrationTest {}}+ /**+ * Runs in its own region so no instance another test left behind already uses {@code default}.+ * AWS creates that group only when an instance first needs it, so before then listing shows+ * just the created group and filtering on {@code default} faults.+ */+ @Test+ void describeListsTheDefaultGroupOnlyOnceAnInstanceUsesIt() {+ String region = "ap-southeast-2";+ dms("CreateReplicationSubnetGroup", region)+ .body("{\"ReplicationSubnetGroupIdentifier\":\"tf-default-probe\","+ + "\"ReplicationSubnetGroupDescription\":\"terraform managed\","+ + "\"SubnetIds\":[\"subnet-default-ap-southeast-2-a\",\"subnet-default-ap-southeast-2-b\"]}")+ .when()+ .post("/")+ .then()+ .statusCode(200);++ dms("DescribeReplicationSubnetGroups", region)+ .body("{}")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("ReplicationSubnetGroups.ReplicationSubnetGroupIdentifier", contains("tf-default-probe"));++ dms("DescribeReplicationSubnetGroups", region)+ .body("{\"Filters\":[{\"Name\":\"replication-subnet-group-id\",\"Values\":[\"default\"]}]}")+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ResourceNotFoundFault"));++ dms("CreateReplicationInstance", region)+ .body("{\"ReplicationInstanceIdentifier\":\"tf-default-user\","+ + "\"ReplicationInstanceClass\":\"dms.t3.micro\"}")+ .when()+ .post("/")+ .then()+ .statusCode(200);++ dms("DescribeReplicationSubnetGroups", region)+ .body("{}")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("ReplicationSubnetGroups.ReplicationSubnetGroupIdentifier",+ contains("default", "tf-default-probe"));++ dms("DescribeReplicationSubnetGroups", region)+ .body("{\"Filters\":[{\"Name\":\"replication-subnet-group-id\",\"Values\":[\"default\"]}]}")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("ReplicationSubnetGroups", hasSize(1))+ .body("ReplicationSubnetGroups[0].ReplicationSubnetGroupIdentifier", equalTo("default"))+ .body("ReplicationSubnetGroups[0].VpcId", equalTo("vpc-default-ap-southeast-2"));+ }+@Testvoid describeRejectsMaxRecordsOutsideTheDocumentedRange() {for (int outOfRange : new int[] {19, 101}) {@@ -373,9 +434,13 @@ class DmsIntegrationTest {}private static RequestSpecification dms(String action) {+ return dms(action, "us-east-1");+ }++ private static RequestSpecification dms(String action, String region) {return given().contentType(CONTENT_TYPE).header("X-Amz-Target", TARGET_PREFIX + action)- .header("Authorization", AUTH_HEADER);+ .header("Authorization", AUTH_HEADER.replace("/us-east-1/", "/" + region + "/"));}}diff --git a/src/test/java/io/github/hectorvent/floci/services/dms/DmsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dms/DmsServiceTest.javaindex 8151bc8057..7c9acfc8d2 100644--- a/src/test/java/io/github/hectorvent/floci/services/dms/DmsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dms/DmsServiceTest.java@@ -33,6 +33,8 @@ import static org.mockito.ArgumentMatchers.anyMap;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;class DmsServiceTest {@@ -42,6 +44,9 @@ class DmsServiceTest {private final ObjectMapper mapper = new ObjectMapper();private DmsService service;+ private Ec2Service ec2Service;+ /** What an unfiltered DescribeSubnets returns; empty means the region has no default VPC. */+ private List<Subnet> regionSubnets = List.of();@BeforeEach@SuppressWarnings({"unchecked", "rawtypes"})@@ -54,9 +59,12 @@ class DmsServiceTest {when(storageFactory.create(eq("dms"), eq("dms-replication-subnet-groups.json"), any(TypeReference.class))).thenReturn((AccountAwareStorageBackend) store);- Ec2Service ec2Service = mock(Ec2Service.class);+ ec2Service = mock(Ec2Service.class);when(ec2Service.describeSubnets(eq(REGION), anyList(), anyMap())).thenAnswer(invocation -> {List<String> requested = invocation.getArgument(1);+ if (requested.isEmpty()) {+ return regionSubnets;+ }return requested.stream().map(DmsServiceTest::subnet).filter(Objects::nonNull).toList();});RegionResolver regionResolver = mock(RegionResolver.class);@@ -151,6 +159,57 @@ class DmsServiceTest {.items().isEmpty());}+ @Test+ void describeFilteredOnDefaultReturnsTheDefaultVpcGroup() {+ useDefaultVpc();+ createInstanceWithoutSubnetGroup();++ List<ReplicationSubnetGroup> described =+ service.describeReplicationSubnetGroups(filterRequest("default"), REGION).items();++ assertEquals(1, described.size());+ ReplicationSubnetGroup group = described.getFirst();+ assertEquals("default", group.getReplicationSubnetGroupIdentifier());+ assertEquals(VPC_ID, group.getVpcId());+ assertEquals(List.of("subnet-a", "subnet-b"), group.getSubnetIds());+ }++ @Test+ void describeListsTheDefaultGroupAlongsideCreatedOnes() {+ useDefaultVpc();+ createInstanceWithoutSubnetGroup();+ service.createReplicationSubnetGroup(createRequest("user-group", "example", "subnet-a", "subnet-b"), REGION);++ List<String> listed = service.describeReplicationSubnetGroups(mapper.createObjectNode(), REGION).items()+ .stream().map(ReplicationSubnetGroup::getReplicationSubnetGroupIdentifier).toList();++ assertEquals(List.of("default", "user-group"), listed);+ }++ @Test+ void describeOmitsTheDefaultGroupUntilAnInstanceUsesIt() {+ useDefaultVpc();+ service.createReplicationSubnetGroup(createRequest("user-group", "example", "subnet-a", "subnet-b"), REGION);++ List<String> listed = service.describeReplicationSubnetGroups(mapper.createObjectNode(), REGION).items()+ .stream().map(ReplicationSubnetGroup::getReplicationSubnetGroupIdentifier).toList();+ assertEquals(List.of("user-group"), listed);+ AwsException notFound = assertThrows(AwsException.class,+ () -> service.describeReplicationSubnetGroups(filterRequest("default"), REGION));+ assertEquals("ResourceNotFoundFault", notFound.getErrorCode());+ // An unfiltered DescribeSubnets is what seeds the region's default VPC in EC2.+ verify(ec2Service, never()).describeSubnets(eq(REGION), eq(List.of()), anyMap());+ }++ @Test+ void describeOmitsTheDefaultGroupWhenTheRegionHasNoDefaultSubnets() {+ createInstanceWithoutSubnetGroup();+ assertTrue(service.describeReplicationSubnetGroups(mapper.createObjectNode(), REGION).items().isEmpty());+ AwsException notFound = assertThrows(AwsException.class,+ () -> service.describeReplicationSubnetGroups(filterRequest("default"), REGION));+ assertEquals("ResourceNotFoundFault", notFound.getErrorCode());+ }+@Testvoid describeOfMissingGroupFaults() {AwsException notFound = assertThrows(AwsException.class,@@ -550,6 +609,21 @@ class DmsServiceTest {return identifiers;}+ private void createInstanceWithoutSubnetGroup() {+ ObjectNode request = mapper.createObjectNode();+ request.put("ReplicationInstanceIdentifier", "uses-default");+ request.put("ReplicationInstanceClass", "dms.t3.micro");+ service.createReplicationInstance(request, REGION);+ }++ private void useDefaultVpc() {+ Subnet a = subnet("subnet-a");+ Subnet b = subnet("subnet-b");+ a.setDefaultForAz(true);+ b.setDefaultForAz(true);+ regionSubnets = List.of(a, b);+ }+private ObjectNode pageRequest(int maxRecords, String marker) {ObjectNode request = mapper.createObjectNode();request.put("MaxRecords", maxRecords);