Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ecs): delete a service's deployment records with the service (#4757)

Allen Sanbornbd0f468d3eae
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index ab63043d68..c93b119c77 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -2731,15 +2731,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
throw new AwsException("InvalidParameterException",
"The service cannot be stopped. Update the service to 0 tasks or use the force flag.", 400);
}
- // Settle against the service as it stands, as UpdateService does: a deployment whose
- // tasks are up has finished, whether or not anything read it.
- currentServiceDeployment(svc);
svc.setStatus("INACTIVE");
- // A deleted service's deployments can never finish, so stop them. This must come before
- // setDesiredCount(0): a concurrent read that saw desiredCount 0 on a still-ACTIVE service
- // would otherwise mark the deployment SUCCESSFUL. AWS deletes the records instead; that
- // is not done here yet.
- stopNonTerminalDeployments(svc.getServiceArn(), Instant.now(), "The service was deleted.");
svc.setDesiredCount(0);
cluster.setActiveServicesCount(Math.max(0, cluster.getActiveServicesCount() - 1));
services.put(key, svc);
@@ -2758,6 +2750,13 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
t.getTaskArn(), e.getMessage());
}
});
+ // AWS deletes the service's deployments and revisions with the service. A service model
+ // restored from the store may have no ARN, and then owns none.
+ String serviceArn = svc.getServiceArn();
+ if (serviceArn != null) {
+ serviceDeployments.values().removeIf(d -> serviceArn.equals(d.getServiceArn()));
+ serviceRevisions.values().removeIf(r -> serviceArn.equals(r.getServiceArn()));
+ }
return svc;
}
@@ -4069,9 +4068,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
}
/**
- * Marks the service's IN_PROGRESS deployments STOPPED, for a caller that has made them
- * unable to finish: a new deployment superseding them, or the service being deleted. Runs
- * before the new deployment is added to the map, so it never stops the current one.
+ * Marks the service's IN_PROGRESS deployments STOPPED, because a new deployment has
+ * superseded them and they can no longer finish. Runs before the new deployment is added to
+ * the map, so it never stops the current one.
*/
private void stopNonTerminalDeployments(String serviceArn, Instant now, String reason) {
// A service model restored from the store may have no ARN.
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsFargateEdgeCaseIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsFargateEdgeCaseIntegrationTest.java
index ae412d1d69..6b593296a0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsFargateEdgeCaseIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsFargateEdgeCaseIntegrationTest.java
@@ -1544,4 +1544,34 @@ class EcsFargateEdgeCaseIntegrationTest {
.body("serviceRevisions", hasSize(0))
.body("failures[0].reason", equalTo("MISSING"));
}
+
+ @Test
+ void deletingAServiceDeletesItsDeploymentsAndRevisions() {
+ String family = seed("edge-deleted-td");
+ call("CreateService", "{\"cluster\":\"" + CLUSTER + "\",\"serviceName\":\"edge-deleted-svc\","
+ + "\"taskDefinition\":\"" + family + "\",\"desiredCount\":0,"
+ + "\"launchType\":\"FARGATE\"," + NETWORK + "}", 200);
+ String list = "{\"cluster\":\"" + CLUSTER + "\",\"service\":\"edge-deleted-svc\"}";
+ Response listed = call("ListServiceDeployments", list, 200);
+ String deploymentArn = listed.jsonPath().getString("serviceDeployments[0].serviceDeploymentArn");
+ String revisionArn = listed.jsonPath().getString("serviceDeployments[0].targetServiceRevisionArn");
+
+ call("DeleteService", list, 200);
+
+ call("ListServiceDeployments", list, 200)
+ .then().body("serviceDeployments", hasSize(0));
+ call("DescribeServiceDeployments", "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}", 200)
+ .then()
+ .body("serviceDeployments", hasSize(0))
+ .body("failures[0].arn", equalTo(deploymentArn))
+ .body("failures[0].reason", equalTo("MISSING"));
+ call("DescribeServiceRevisions", "{\"serviceRevisionArns\":[\"" + revisionArn + "\"]}", 200)
+ .then()
+ .body("serviceRevisions", hasSize(0))
+ .body("failures[0].reason", equalTo("MISSING"));
+ call("DescribeServices", "{\"cluster\":\"" + CLUSTER + "\",\"services\":[\"edge-deleted-svc\"]}", 200)
+ .then()
+ .body("services[0].status", equalTo("INACTIVE"))
+ .body("services[0].currentServiceDeployment", nullValue());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeleteDeploymentsTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeleteDeploymentsTest.java
new file mode 100644
index 0000000000..52b26936dd
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeleteDeploymentsTest.java
@@ -0,0 +1,132 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;
+import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import io.github.hectorvent.floci.services.ecs.model.ServiceDeployment;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * "Amazon ECS deletes the service deployment when you delete a service", and likewise the service
+ * revision. A deleted service must not keep them, nor hand them to a service later created under
+ * the same name.
+ */
+class EcsServiceDeleteDeploymentsTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String CLUSTER = "del-cluster";
+
+ @Test
+ void deleteServiceRemovesItsDeploymentsAndRevisions() {
+ EcsService service = newMockModeService();
+ service.createCluster(CLUSTER, REGION);
+ registerTaskDef(service);
+ service.createService(CLUSTER, "doomed", "del-fam", 1, LaunchType.FARGATE, List.of(), null, REGION);
+ service.createService(CLUSTER, "kept", "del-fam", 1, LaunchType.FARGATE, List.of(), null, REGION);
+ service.reconcileServices();
+
+ ServiceDeployment doomed = onlyDeployment(service, "doomed");
+ ServiceDeployment kept = onlyDeployment(service, "kept");
+
+ service.deleteService(CLUSTER, "doomed", true, REGION);
+ service.reconcileServices();
+
+ assertEquals(List.of(), service.listServiceDeployments("doomed", CLUSTER, null, REGION),
+ "a deleted service lists no deployments");
+ assertEquals(List.of(), service.describeServiceDeployments(List.of(doomed.getServiceDeploymentArn())),
+ "the deleted deployment's ARN no longer resolves");
+ assertEquals(List.of(), service.describeServiceRevisions(List.of(doomed.getTargetServiceRevisionArn())),
+ "the deleted service's revision no longer resolves");
+ EcsServiceModel inactive = service.describeServices(CLUSTER, List.of("doomed"), REGION).getFirst();
+ assertEquals("INACTIVE", inactive.getStatus());
+ assertNull(service.currentServiceDeployment(inactive),
+ "DescribeServices on the deleted service points at no deployment");
+
+ assertEquals(List.of(kept.getServiceDeploymentArn()),
+ service.listServiceDeployments("kept", CLUSTER, null, REGION),
+ "another service in the cluster keeps its deployment");
+ assertEquals(1, service.describeServiceRevisions(List.of(kept.getTargetServiceRevisionArn())).size(),
+ "another service in the cluster keeps its revision");
+ }
+
+ @Test
+ void aRecreatedServiceDoesNotInheritTheDeletedOnesDeployments() {
+ EcsService service = newMockModeService();
+ service.createCluster(CLUSTER, REGION);
+ registerTaskDef(service);
+ service.createService(CLUSTER, "again", "del-fam", 0, LaunchType.FARGATE, List.of(), null, REGION);
+ ServiceDeployment first = onlyDeployment(service, "again");
+ service.deleteService(CLUSTER, "again", false, REGION);
+
+ service.createService(CLUSTER, "again", "del-fam", 0, LaunchType.FARGATE, List.of(), null, REGION);
+ service.reconcileServices();
+
+ ServiceDeployment second = onlyDeployment(service, "again");
+ assertEquals(first.getServiceArn(), second.getServiceArn(), "the recreated service reuses the ARN");
+ assertEquals(List.of(), second.getSourceServiceRevisionArns(),
+ "the new deployment does not roll from the deleted service's revision");
+ }
+
+ private static ServiceDeployment onlyDeployment(EcsService service, String serviceName) {
+ List<ServiceDeployment> deployments =
+ service.listServiceDeploymentsDetailed(serviceName, CLUSTER, null, REGION);
+ assertEquals(1, deployments.size(), serviceName + " has exactly one deployment");
+ return deployments.getFirst();
+ }
+
+ private static void registerTaskDef(EcsService service) {
+ ContainerDefinition cd = new ContainerDefinition();
+ cd.setName("app");
+ cd.setImage("app:1");
+ service.registerTaskDefinition("del-fam", List.of(cd), null, null, null, null, null, List.of(), REGION);
+ }
+
+ private static EcsService newMockModeService() {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().mock()).thenReturn(true);
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ EcsService service = new EcsService(
+ new RegionResolver(REGION, "000000000000"),
+ mock(EcsContainerManager.class),
+ config,
+ mock(EcsLoadBalancerRegistrar.class),
+ new InMemoryStorageFactory(),
+ null);
+ service.initializeStorage();
+ return service;
+ }
+
+ private static final class InMemoryStorageFactory extends StorageFactory {
+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
+
+ private InMemoryStorageFactory() {
+ super(null, null);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> AccountAwareStorageBackend<V> create(String serviceName,
+ String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,
+ ignored -> AccountAwareStorageBackend.inMemory("000000000000"));
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentStatusTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentStatusTest.java
index 4eff9cdef0..34ae2a8c22 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentStatusTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeploymentStatusTest.java
@@ -429,89 +429,6 @@ class EcsServiceDeploymentStatusTest {
"only the new deployment is in progress");
}
- /**
- * Deleting a service stops a deployment it leaves unfinished, so a listing filtered on
- * IN_PROGRESS no longer returns it.
- */
- @Test
- void deletingAServiceStopsTheDeploymentItLeavesUnfinished() {
- EcsService service = newMockModeService();
- service.createCluster("ddel-cluster", REGION);
- registerTaskDef(service, "ddel-fam", "app:1");
- String only = service.createService("ddel-cluster", "ddel-svc", "ddel-fam", 1,
- LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();
-
- // No tick, so nothing was ever launched and the deployment is still rolling out.
- assertEquals("IN_PROGRESS",
- deploymentOf(service, "ddel-svc", "ddel-cluster", only).getStatus(),
- "precondition: the deployment never converged");
- assertEquals(1, service.listServiceDeployments("ddel-svc", "ddel-cluster",
- List.of("IN_PROGRESS"), REGION).size(), "precondition: and reads as in flight");
-
- service.deleteService("ddel-cluster", "ddel-svc", true, REGION);
-
- ServiceDeployment stopped = deploymentOf(service, "ddel-svc", "ddel-cluster", only);
- assertEquals("STOPPED", stopped.getStatus(),
- "a deployment whose service is gone has ended, and nothing else can ever say so");
- assertNotNull(stopped.getFinishedAt(), "it reports when it ended");
- assertNotNull(stopped.getStoppedAt(), "and when it was stopped");
- assertEquals("The service was deleted.", stopped.getStatusReason(),
- "naming the reason it could not finish");
-
- assertTrue(service.listServiceDeployments("ddel-svc", "ddel-cluster",
- List.of("IN_PROGRESS"), REGION).isEmpty(),
- "and a client asking what is rolling out is no longer told this is");
- }
-
- /** Deleting a service leaves its finished deployments SUCCESSFUL. */
- @Test
- void deletingAServiceLeavesItsFinishedDeploymentsSuccessful() {
- EcsService service = newMockModeService();
- service.createCluster("ddel2-cluster", REGION);
- registerTaskDef(service, "ddel2-fam", "app:1");
- String only = service.createService("ddel2-cluster", "ddel2-svc", "ddel2-fam", 1,
- LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();
- service.reconcileServices();
-
- ServiceDeployment finished = deploymentOf(service, "ddel2-svc", "ddel2-cluster", only);
- assertEquals("SUCCESSFUL", finished.getStatus(), "precondition: it converged and was read");
- Instant finishedAt = finished.getFinishedAt();
- assertNotNull(finishedAt);
-
- service.deleteService("ddel2-cluster", "ddel2-svc", true, REGION);
-
- ServiceDeployment after = deploymentOf(service, "ddel2-svc", "ddel2-cluster", only);
- assertEquals("SUCCESSFUL", after.getStatus(),
- "a rollout that completed is history, and deleting the service does not undo it");
- assertEquals(finishedAt, after.getFinishedAt(), "keeping the instant it finished at");
- assertNull(after.getStoppedAt(), "and carrying none of the stopped fields");
- assertNull(after.getStatusReason());
- }
-
- /**
- * As {@link #aConvergedDeploymentNobodyReadIsSuccessfulWhenSuperseded}, for a delete: the
- * delete settles the deployment against the service as it stood, so a rollout that had
- * finished is SUCCESSFUL whether or not anything read it first.
- */
- @Test
- void deletingAServiceSettlesAConvergedDeploymentNobodyRead() {
- EcsService service = newMockModeService();
- service.createCluster("ddel3-cluster", REGION);
- registerTaskDef(service, "ddel3-fam", "app:1");
- String only = service.createService("ddel3-cluster", "ddel3-svc", "ddel3-fam", 1,
- LaunchType.FARGATE, List.of(), null, REGION).getDeploymentId();
- service.reconcileServices();
-
- service.deleteService("ddel3-cluster", "ddel3-svc", true, REGION);
-
- ServiceDeployment after = deploymentOf(service, "ddel3-svc", "ddel3-cluster", only);
- assertEquals("SUCCESSFUL", after.getStatus(),
- "its task was running when the delete arrived");
- assertNotNull(after.getFinishedAt());
- assertNull(after.getStoppedAt(), "a deployment that succeeded was never stopped");
- assertNull(after.getStatusReason());
- }
-
/** The one place the provider's join is asserted: the revision ARN carries the task set id. */
private static void assertRevisionCarriesTaskSetId(EcsService service, String name,
String cluster, String deploymentId) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java
index d4cff003bf..b11778a8cd 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java
@@ -13,6 +13,7 @@ import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.startsWith;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@@ -161,14 +162,14 @@ class EcsServiceSteadyStateWaiterIntegrationTest {
* {@code ServiceDeployment} shape and neither of which floci used to write at all, so a
* client asking when a stopped deployment ended, or why, got nothing.
*
- * <p>The deployment has to still be in flight when the delete arrives, and a test cannot
- * promise that: the 5-second reconciler runs on its own thread, and a tick that lands between
- * CreateService and DeleteService starts the task, so the delete correctly settles the
+ * <p>The deployment has to still be in flight when the update supersedes it, and a test
+ * cannot promise that: the 5-second reconciler runs on its own thread, and a tick that lands
+ * between CreateService and UpdateService starts the task, so the update correctly settles the
* deployment SUCCESSFUL instead. That is a lost race, not a failure, so the test tries again
* on a fresh service, a few times; the window is milliseconds against a 5000ms period. It
* fails only if no attempt produced a stopped deployment, or one did and its fields are wrong.
* The behaviour itself is pinned without any timing in
- * {@code EcsServiceDeploymentStatusTest.deletingAServiceStopsTheDeploymentItLeavesUnfinished};
+ * {@code EcsServiceDeploymentStatusTest.supersedingAnUnconvergedDeploymentStopsIt};
* what only this test can show is that the two fields reach the wire.
*/
@Test
@@ -176,31 +177,30 @@ class EcsServiceSteadyStateWaiterIntegrationTest {
for (int attempt = 1; attempt <= 5; attempt++) {
String service = seedUnconvergedService("waiter-stop-svc-" + attempt);
- call("DeleteService", "{\"cluster\":\"" + CLUSTER + "\",\"service\":\"" + service
- + "\",\"force\":true}");
+ call("UpdateService", "{\"cluster\":\"" + CLUSTER + "\",\"service\":\"" + service
+ + "\",\"forceNewDeployment\":true}");
- // Identified after the delete: the service was created once and never updated, so it
- // has exactly one deployment, and a terminal record cannot change under a read.
- Response listed = call("ListServiceDeployments", "{\"cluster\":\"" + CLUSTER
- + "\",\"service\":\"" + service + "\"}");
- listed.then().body("serviceDeployments", hasSize(1));
- String deploymentArn = listed.jsonPath()
- .getString("serviceDeployments[0].serviceDeploymentArn");
- assertNotNull(deploymentArn, "the deleted service's deployment is still describable");
-
- Response described = call("DescribeServiceDeployments",
- "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}");
- if ("SUCCESSFUL".equals(described.jsonPath().getString("serviceDeployments[0].status"))) {
+ // A terminal record cannot change under a read, so filter for it rather than race.
+ Response stopped = call("ListServiceDeployments", "{\"cluster\":\"" + CLUSTER
+ + "\",\"service\":\"" + service + "\",\"status\":[\"STOPPED\"]}");
+ if (stopped.jsonPath().getList("serviceDeployments").isEmpty()) {
continue;
}
- described.then()
+ stopped.then().body("serviceDeployments", hasSize(1));
+ String deploymentArn = stopped.jsonPath()
+ .getString("serviceDeployments[0].serviceDeploymentArn");
+
+ call("DescribeServiceDeployments",
+ "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}")
+ .then()
.body("serviceDeployments", hasSize(1))
.body("serviceDeployments[0].status", equalTo("STOPPED"))
.body("serviceDeployments[0].stoppedAt", notNullValue())
.body("serviceDeployments[0].finishedAt", notNullValue())
- .body("serviceDeployments[0].statusReason", equalTo("The service was deleted."));
+ .body("serviceDeployments[0].statusReason",
+ startsWith("Superseded by service deployment "));
return;
}
- fail("the reconciler converged the service before every one of five deletes");
+ fail("the reconciler converged the service before every one of five updates");
}
}