Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ecs): delete a service's deployment records with the service (#4757)
Allen Sanbornbd0f468d3eae
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.javaindex ab63043d68..c93b119c77 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java@@ -2731,15 +2731,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabthrow new AwsException("InvalidParameterException","The service cannot be stopped. Update the service to 0 tasks or use the force flag.", 400);}- // Settle against the service as it stands, as UpdateService does: a deployment whose- // tasks are up has finished, whether or not anything read it.- currentServiceDeployment(svc);svc.setStatus("INACTIVE");- // A deleted service's deployments can never finish, so stop them. This must come before- // setDesiredCount(0): a concurrent read that saw desiredCount 0 on a still-ACTIVE service- // would otherwise mark the deployment SUCCESSFUL. AWS deletes the records instead; that- // is not done here yet.- stopNonTerminalDeployments(svc.getServiceArn(), Instant.now(), "The service was deleted.");svc.setDesiredCount(0);cluster.setActiveServicesCount(Math.max(0, cluster.getActiveServicesCount() - 1));services.put(key, svc);@@ -2758,6 +2750,13 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabt.getTaskArn(), e.getMessage());}});+ // AWS deletes the service's deployments and revisions with the service. A service model+ // restored from the store may have no ARN, and then owns none.+ String serviceArn = svc.getServiceArn();+ if (serviceArn != null) {+ serviceDeployments.values().removeIf(d -> serviceArn.equals(d.getServiceArn()));+ serviceRevisions.values().removeIf(r -> serviceArn.equals(r.getServiceArn()));+ }return svc;}@@ -4069,9 +4068,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab}/**- * Marks the service's IN_PROGRESS deployments STOPPED, for a caller that has made them- * unable to finish: a new deployment superseding them, or the service being deleted. Runs- * before the new deployment is added to the map, so it never stops the current one.+ * Marks the service's IN_PROGRESS deployments STOPPED, because a new deployment has+ * superseded them and they can no longer finish. Runs before the new deployment is added to+ * the map, so it never stops the current one.*/private void stopNonTerminalDeployments(String serviceArn, Instant now, String reason) {// A service model restored from the store may have no ARN.diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsFargateEdgeCaseIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsFargateEdgeCaseIntegrationTest.javaindex ae412d1d69..6b593296a0 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsFargateEdgeCaseIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsFargateEdgeCaseIntegrationTest.java@@ -1544,4 +1544,34 @@ class EcsFargateEdgeCaseIntegrationTest {.body("serviceRevisions", hasSize(0)).body("failures[0].reason", equalTo("MISSING"));}++ @Test+ void deletingAServiceDeletesItsDeploymentsAndRevisions() {+ String family = seed("edge-deleted-td");+ call("CreateService", "{\"cluster\":\"" + CLUSTER + "\",\"serviceName\":\"edge-deleted-svc\","+ + "\"taskDefinition\":\"" + family + "\",\"desiredCount\":0,"+ + "\"launchType\":\"FARGATE\"," + NETWORK + "}", 200);+ String list = "{\"cluster\":\"" + CLUSTER + "\",\"service\":\"edge-deleted-svc\"}";+ Response listed = call("ListServiceDeployments", list, 200);+ String deploymentArn = listed.jsonPath().getString("serviceDeployments[0].serviceDeploymentArn");+ String revisionArn = listed.jsonPath().getString("serviceDeployments[0].targetServiceRevisionArn");++ call("DeleteService", list, 200);++ call("ListServiceDeployments", list, 200)+ .then().body("serviceDeployments", hasSize(0));+ call("DescribeServiceDeployments", "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}", 200)+ .then()+ .body("serviceDeployments", hasSize(0))+ .body("failures[0].arn", equalTo(deploymentArn))+ .body("failures[0].reason", equalTo("MISSING"));+ call("DescribeServiceRevisions", "{\"serviceRevisionArns\":[\"" + revisionArn + "\"]}", 200)+ .then()+ .body("serviceRevisions", hasSize(0))+ .body("failures[0].reason", equalTo("MISSING"));+ call("DescribeServices", "{\"cluster\":\"" + CLUSTER + "\",\"services\":[\"edge-deleted-svc\"]}", 200)+ .then()+ .body("services[0].status", equalTo("INACTIVE"))+ .body("services[0].currentServiceDeployment", nullValue());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeleteDeploymentsTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeleteDeploymentsTest.javanew file mode 100644index 0000000000..52b26936dd--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeleteDeploymentsTest.java@@ -0,0 +1,132 @@+package io.github.hectorvent.floci.services.ecs;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;+import io.github.hectorvent.floci.services.ecs.model.LaunchType;+import io.github.hectorvent.floci.services.ecs.model.ServiceDeployment;+import org.junit.jupiter.api.Test;++import java.util.HashMap;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++/**+ * "Amazon ECS deletes the service deployment when you delete a service", and likewise the service+ * revision. A deleted service must not keep them, nor hand them to a service later created under+ * the same name.+ */+class EcsServiceDeleteDeploymentsTest {++ private static final String REGION = "us-east-1";+ private static final String CLUSTER = "del-cluster";++ @Test+ void deleteServiceRemovesItsDeploymentsAndRevisions() {+ EcsService service = newMockModeService();+ service.createCluster(CLUSTER, REGION);+ registerTaskDef(service);+ service.createService(CLUSTER, "doomed", "del-fam", 1, LaunchType.FARGATE, List.of(), null, REGION);+ service.createService(CLUSTER, "kept", "del-fam", 1, LaunchType.FARGATE, List.of(), null, REGION);+ service.reconcileServices();++ ServiceDeployment doomed = onlyDeployment(service, "doomed");+ ServiceDeployment kept = onlyDeployment(service, "kept");++ service.deleteService(CLUSTER, "doomed", true, REGION);+ service.reconcileServices();++ assertEquals(List.of(), service.listServiceDeployments("doomed", CLUSTER, null, REGION),+ "a deleted service lists no deployments");+ assertEquals(List.of(), service.describeServiceDeployments(List.of(doomed.getServiceDeploymentArn())),+ "the deleted deployment's ARN no longer resolves");+ assertEquals(List.of(), service.describeServiceRevisions(List.of(doomed.getTargetServiceRevisionArn())),+ "the deleted service's revision no longer resolves");+ EcsServiceModel inactive = service.describeServices(CLUSTER, List.of("doomed"), REGION).getFirst();+ assertEquals("INACTIVE", inactive.getStatus());+ assertNull(service.currentServiceDeployment(inactive),+ "DescribeServices on the deleted service points at no deployment");++ assertEquals(List.of(kept.getServiceDeploymentArn()),+ service.listServiceDeployments("kept", CLUSTER, null, REGION),+ "another service in the cluster keeps its deployment");+ assertEquals(1, service.describeServiceRevisions(List.of(kept.getTargetServiceRevisionArn())).size(),+ "another service in the cluster keeps its revision");+ }++ @Test+ void aRecreatedServiceDoesNotInheritTheDeletedOnesDeployments() {+ EcsService service = newMockModeService();+ service.createCluster(CLUSTER, REGION);+ registerTaskDef(service);+ service.createService(CLUSTER, "again", "del-fam", 0, LaunchType.FARGATE, List.of(), null, REGION);+ ServiceDeployment first = onlyDeployment(service, "again");+ service.deleteService(CLUSTER, "again", false, REGION);++ service.createService(CLUSTER, "again", "del-fam", 0, LaunchType.FARGATE, List.of(), null, REGION);+ service.reconcileServices();++ ServiceDeployment second = onlyDeployment(service, "again");+ assertEquals(first.getServiceArn(), second.getServiceArn(), "the recreated service reuses the ARN");+ assertEquals(List.of(), second.getSourceServiceRevisionArns(),+ "the new deployment does not roll from the deleted service's revision");+ }++ private static ServiceDeployment onlyDeployment(EcsService service, String serviceName) {+ List<ServiceDeployment> deployments =+ service.listServiceDeploymentsDetailed(serviceName, CLUSTER, null, REGION);+ assertEquals(1, deployments.size(), serviceName + " has exactly one deployment");+ return deployments.getFirst();+ }++ private static void registerTaskDef(EcsService service) {+ ContainerDefinition cd = new ContainerDefinition();+ cd.setName("app");+ cd.setImage("app:1");+ service.registerTaskDefinition("del-fam", List.of(cd), null, null, null, null, null, List.of(), REGION);+ }++ private static EcsService newMockModeService() {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().ecs().mock()).thenReturn(true);+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ EcsService service = new EcsService(+ new RegionResolver(REGION, "000000000000"),+ mock(EcsContainerManager.class),+ config,+ mock(EcsLoadBalancerRegistrar.class),+ new InMemoryStorageFactory(),+ null);+ service.initializeStorage();+ return service;+ }++ private static final class InMemoryStorageFactory extends StorageFactory {+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();++ private InMemoryStorageFactory() {+ super(null, null);+ }++ @Override+ @SuppressWarnings("unchecked")+ public <V> AccountAwareStorageBackend<V> create(String serviceName,+ String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,+ ignored -> AccountAwareStorageBackend.inMemory("000000000000"));+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentStatusTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentStatusTest.javaindex 4eff9cdef0..34ae2a8c22 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentStatusTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentStatusTest.java@@ -429,89 +429,6 @@ class EcsServiceDeploymentStatusTest {"only the new deployment is in progress");}- /**- * Deleting a service stops a deployment it leaves unfinished, so a listing filtered on- * IN_PROGRESS no longer returns it.- */- @Test- void deletingAServiceStopsTheDeploymentItLeavesUnfinished() {- EcsService service = newMockModeService();- service.createCluster("ddel-cluster", REGION);- registerTaskDef(service, "ddel-fam", "app:1");- String only = service.createService("ddel-cluster", "ddel-svc", "ddel-fam", 1,- LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();-- // No tick, so nothing was ever launched and the deployment is still rolling out.- assertEquals("IN_PROGRESS",- deploymentOf(service, "ddel-svc", "ddel-cluster", only).getStatus(),- "precondition: the deployment never converged");- assertEquals(1, service.listServiceDeployments("ddel-svc", "ddel-cluster",- List.of("IN_PROGRESS"), REGION).size(), "precondition: and reads as in flight");-- service.deleteService("ddel-cluster", "ddel-svc", true, REGION);-- ServiceDeployment stopped = deploymentOf(service, "ddel-svc", "ddel-cluster", only);- assertEquals("STOPPED", stopped.getStatus(),- "a deployment whose service is gone has ended, and nothing else can ever say so");- assertNotNull(stopped.getFinishedAt(), "it reports when it ended");- assertNotNull(stopped.getStoppedAt(), "and when it was stopped");- assertEquals("The service was deleted.", stopped.getStatusReason(),- "naming the reason it could not finish");-- assertTrue(service.listServiceDeployments("ddel-svc", "ddel-cluster",- List.of("IN_PROGRESS"), REGION).isEmpty(),- "and a client asking what is rolling out is no longer told this is");- }-- /** Deleting a service leaves its finished deployments SUCCESSFUL. */- @Test- void deletingAServiceLeavesItsFinishedDeploymentsSuccessful() {- EcsService service = newMockModeService();- service.createCluster("ddel2-cluster", REGION);- registerTaskDef(service, "ddel2-fam", "app:1");- String only = service.createService("ddel2-cluster", "ddel2-svc", "ddel2-fam", 1,- LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();- service.reconcileServices();-- ServiceDeployment finished = deploymentOf(service, "ddel2-svc", "ddel2-cluster", only);- assertEquals("SUCCESSFUL", finished.getStatus(), "precondition: it converged and was read");- Instant finishedAt = finished.getFinishedAt();- assertNotNull(finishedAt);-- service.deleteService("ddel2-cluster", "ddel2-svc", true, REGION);-- ServiceDeployment after = deploymentOf(service, "ddel2-svc", "ddel2-cluster", only);- assertEquals("SUCCESSFUL", after.getStatus(),- "a rollout that completed is history, and deleting the service does not undo it");- assertEquals(finishedAt, after.getFinishedAt(), "keeping the instant it finished at");- assertNull(after.getStoppedAt(), "and carrying none of the stopped fields");- assertNull(after.getStatusReason());- }-- /**- * As {@link #aConvergedDeploymentNobodyReadIsSuccessfulWhenSuperseded}, for a delete: the- * delete settles the deployment against the service as it stood, so a rollout that had- * finished is SUCCESSFUL whether or not anything read it first.- */- @Test- void deletingAServiceSettlesAConvergedDeploymentNobodyRead() {- EcsService service = newMockModeService();- service.createCluster("ddel3-cluster", REGION);- registerTaskDef(service, "ddel3-fam", "app:1");- String only = service.createService("ddel3-cluster", "ddel3-svc", "ddel3-fam", 1,- LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();- service.reconcileServices();-- service.deleteService("ddel3-cluster", "ddel3-svc", true, REGION);-- ServiceDeployment after = deploymentOf(service, "ddel3-svc", "ddel3-cluster", only);- assertEquals("SUCCESSFUL", after.getStatus(),- "its task was running when the delete arrived");- assertNotNull(after.getFinishedAt());- assertNull(after.getStoppedAt(), "a deployment that succeeded was never stopped");- assertNull(after.getStatusReason());- }-/** The one place the provider's join is asserted: the revision ARN carries the task set id. */private static void assertRevisionCarriesTaskSetId(EcsService service, String name,String cluster, String deploymentId) {diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.javaindex d4cff003bf..b11778a8cd 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java@@ -13,6 +13,7 @@ import static org.hamcrest.Matchers.equalTo;import static org.hamcrest.Matchers.hasSize;import static org.hamcrest.Matchers.notNullValue;import static org.hamcrest.Matchers.nullValue;+import static org.hamcrest.Matchers.startsWith;import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.junit.jupiter.api.Assertions.fail;@@ -161,14 +162,14 @@ class EcsServiceSteadyStateWaiterIntegrationTest {* {@code ServiceDeployment} shape and neither of which floci used to write at all, so a* client asking when a stopped deployment ended, or why, got nothing.*- * <p>The deployment has to still be in flight when the delete arrives, and a test cannot- * promise that: the 5-second reconciler runs on its own thread, and a tick that lands between- * CreateService and DeleteService starts the task, so the delete correctly settles the+ * <p>The deployment has to still be in flight when the update supersedes it, and a test+ * cannot promise that: the 5-second reconciler runs on its own thread, and a tick that lands+ * between CreateService and UpdateService starts the task, so the update correctly settles the* deployment SUCCESSFUL instead. That is a lost race, not a failure, so the test tries again* on a fresh service, a few times; the window is milliseconds against a 5000ms period. It* fails only if no attempt produced a stopped deployment, or one did and its fields are wrong.* The behaviour itself is pinned without any timing in- * {@code EcsServiceDeploymentStatusTest.deletingAServiceStopsTheDeploymentItLeavesUnfinished};+ * {@code EcsServiceDeploymentStatusTest.supersedingAnUnconvergedDeploymentStopsIt};* what only this test can show is that the two fields reach the wire.*/@Test@@ -176,31 +177,30 @@ class EcsServiceSteadyStateWaiterIntegrationTest {for (int attempt = 1; attempt <= 5; attempt++) {String service = seedUnconvergedService("waiter-stop-svc-" + attempt);- call("DeleteService", "{\"cluster\":\"" + CLUSTER + "\",\"service\":\"" + service- + "\",\"force\":true}");+ call("UpdateService", "{\"cluster\":\"" + CLUSTER + "\",\"service\":\"" + service+ + "\",\"forceNewDeployment\":true}");- // Identified after the delete: the service was created once and never updated, so it- // has exactly one deployment, and a terminal record cannot change under a read.- Response listed = call("ListServiceDeployments", "{\"cluster\":\"" + CLUSTER- + "\",\"service\":\"" + service + "\"}");- listed.then().body("serviceDeployments", hasSize(1));- String deploymentArn = listed.jsonPath()- .getString("serviceDeployments[0].serviceDeploymentArn");- assertNotNull(deploymentArn, "the deleted service's deployment is still describable");-- Response described = call("DescribeServiceDeployments",- "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}");- if ("SUCCESSFUL".equals(described.jsonPath().getString("serviceDeployments[0].status"))) {+ // A terminal record cannot change under a read, so filter for it rather than race.+ Response stopped = call("ListServiceDeployments", "{\"cluster\":\"" + CLUSTER+ + "\",\"service\":\"" + service + "\",\"status\":[\"STOPPED\"]}");+ if (stopped.jsonPath().getList("serviceDeployments").isEmpty()) {continue;}- described.then()+ stopped.then().body("serviceDeployments", hasSize(1));+ String deploymentArn = stopped.jsonPath()+ .getString("serviceDeployments[0].serviceDeploymentArn");++ call("DescribeServiceDeployments",+ "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}")+ .then().body("serviceDeployments", hasSize(1)).body("serviceDeployments[0].status", equalTo("STOPPED")).body("serviceDeployments[0].stoppedAt", notNullValue()).body("serviceDeployments[0].finishedAt", notNullValue())- .body("serviceDeployments[0].statusReason", equalTo("The service was deleted."));+ .body("serviceDeployments[0].statusReason",+ startsWith("Superseded by service deployment "));return;}- fail("the reconciler converged the service before every one of five deletes");+ fail("the reconciler converged the service before every one of five updates");}}