Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ecs): serialize create, update and delete of one service (#5845)

Allen Sanbornf031e4052d13
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index 6e386c8434..837b7b9b2b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -217,6 +217,12 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
private Map<String, CapacityProvider> capacityProviders = new ConcurrentHashMap<>();
// taskSetArn → TaskSet
private final Map<String, TaskSet> taskSets = new ConcurrentHashMap<>();
+ // Serializes create/update/delete of one service key (region, cluster, name). A fixed set of
+ // stripes, never cleared, so unknown or failed names retain nothing and a state reset cannot
+ // hand a same-key call a different lock than one still held by an in-flight call.
+ // ponytail: distinct services sharing a stripe serialize too; widen if contention shows.
+ private static final int SERVICE_LOCK_STRIPES = 64;
+ private final Object[] serviceLocks = newServiceLocks();
// serviceDeploymentArn → ServiceDeployment
private final Map<String, ServiceDeployment> serviceDeployments = new ConcurrentHashMap<>();
// serviceRevisionArn → ServiceRevision
@@ -2576,86 +2582,88 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
String serviceName = request.getServiceName();
String key = serviceKey(region, cluster.getClusterName(), serviceName);
- if (services.containsKey(key)) {
- EcsServiceModel existingSvc = services.get(key);
+ synchronized (serviceLock(key)) {
+ if (services.containsKey(key)) {
+ EcsServiceModel existingSvc = services.get(key);
- if ("ACTIVE".equals(existingSvc.getStatus())) {
+ if ("ACTIVE".equals(existingSvc.getStatus())) {
+ throw new AwsException("InvalidParameterException",
+ "Creation of service was not idempotent.", 400);
+ }
+ }
+ if (request.getLaunchType() != null && request.getCapacityProviderStrategy() != null
+ && !request.getCapacityProviderStrategy().isEmpty()) {
throw new AwsException("InvalidParameterException",
- "Creation of service was not idempotent.", 400);
+ "You cannot specify both a launch type and a capacity provider strategy in the "
+ + "same request. Specify only one.", 400);
+ }
+ validateLaunchNetworking(taskDef, request.getNetworkConfiguration());
+
+ EcsServiceModel svc = new EcsServiceModel();
+ svc.setServiceArn(regionResolver.buildArn("ecs", region,
+ "service/" + cluster.getClusterName() + "/" + serviceName));
+ svc.setServiceName(serviceName);
+ svc.setClusterArn(cluster.getClusterArn());
+ svc.setTaskDefinition(taskDefinition);
+ svc.setCapacityProviderStrategy(request.getCapacityProviderStrategy());
+ svc.setLaunchType(serviceLaunchType(cluster, request));
+ int desiredCount = request.getDesiredCount();
+ if (desiredCount < 0) {
+ throw new AwsException("InvalidParameterException", "desiredCount cannot be a negative number.", 400);
+ }
+ svc.setDesiredCount(desiredCount);
+ svc.setLoadBalancers(request.getLoadBalancers());
+ svc.setServiceRegistries(request.getServiceRegistries());
+ svc.setNetworkConfiguration(request.getNetworkConfiguration());
+ // AWS echoes these on every DescribeServices; clients that persist them (Terraform's
+ // aws_ecs_service reads all three, and schedulingStrategy is ForceNew) treat a missing
+ // value as drift and replace the service on every apply.
+ String strategy = request.getSchedulingStrategy() != null
+ ? request.getSchedulingStrategy() : DEFAULT_SCHEDULING_STRATEGY;
+ String controller = request.getDeploymentControllerType() != null
+ ? request.getDeploymentControllerType() : DEFAULT_DEPLOYMENT_CONTROLLER;
+ if (SCHEDULING_DAEMON.equals(strategy)
+ && (svc.getLaunchType() == LaunchType.FARGATE || !DEFAULT_DEPLOYMENT_CONTROLLER.equals(controller))) {
+ throw new AwsException("InvalidParameterException",
+ "Tasks using the Fargate launch type or the CODE_DEPLOY or EXTERNAL deployment "
+ + "controller types don't support the DAEMON scheduling strategy.", 400);
+ }
+ svc.setSchedulingStrategy(strategy);
+ svc.setDeploymentController(controller);
+ svc.setAvailabilityZoneRebalancing(request.getAvailabilityZoneRebalancing() != null
+ ? request.getAvailabilityZoneRebalancing() : DEFAULT_AZ_REBALANCING_ON_CREATE);
+ svc.setServiceConnectConfiguration(request.getServiceConnectConfiguration());
+ svc.setDeploymentConfiguration(request.getDeploymentConfiguration());
+ svc.setEnableExecuteCommand(request.isEnableExecuteCommand());
+ svc.setEnableECSManagedTags(request.isEnableECSManagedTags());
+ // Both have documented defaults a created service reports back.
+ svc.setPropagateTags(request.getPropagateTags() != null
+ ? request.getPropagateTags() : PROPAGATE_TAGS_NONE);
+ svc.setHealthCheckGracePeriodSeconds(request.getHealthCheckGracePeriodSeconds() != null
+ ? request.getHealthCheckGracePeriodSeconds() : 0);
+ validateServiceRole(request, taskDef);
+ svc.setRoleArn(request.getRoleArn());
+ svc.setUnparsed(request.getUnparsed());
+ applyServicePlatform(svc, taskDef, request.getPlatformVersion());
+ svc.setStatus("ACTIVE");
+ svc.setCreatedAt(Instant.now());
+ svc.setLastDeploymentAt(svc.getCreatedAt());
+ svc.setDeploymentId(newDeploymentId());
+ if (request.getTags() != null && !request.getTags().isEmpty()) {
+ svc.setTags(new LinkedHashMap<>(request.getTags()));
}
- }
- if (request.getLaunchType() != null && request.getCapacityProviderStrategy() != null
- && !request.getCapacityProviderStrategy().isEmpty()) {
- throw new AwsException("InvalidParameterException",
- "You cannot specify both a launch type and a capacity provider strategy in the "
- + "same request. Specify only one.", 400);
- }
- validateLaunchNetworking(taskDef, request.getNetworkConfiguration());
-
- EcsServiceModel svc = new EcsServiceModel();
- svc.setServiceArn(regionResolver.buildArn("ecs", region,
- "service/" + cluster.getClusterName() + "/" + serviceName));
- svc.setServiceName(serviceName);
- svc.setClusterArn(cluster.getClusterArn());
- svc.setTaskDefinition(taskDefinition);
- svc.setCapacityProviderStrategy(request.getCapacityProviderStrategy());
- svc.setLaunchType(serviceLaunchType(cluster, request));
- int desiredCount = request.getDesiredCount();
- if (desiredCount < 0) {
- throw new AwsException("InvalidParameterException", "desiredCount cannot be a negative number.", 400);
- }
- svc.setDesiredCount(desiredCount);
- svc.setLoadBalancers(request.getLoadBalancers());
- svc.setServiceRegistries(request.getServiceRegistries());
- svc.setNetworkConfiguration(request.getNetworkConfiguration());
- // AWS echoes these on every DescribeServices; clients that persist them (Terraform's
- // aws_ecs_service reads all three, and schedulingStrategy is ForceNew) treat a missing
- // value as drift and replace the service on every apply.
- String strategy = request.getSchedulingStrategy() != null
- ? request.getSchedulingStrategy() : DEFAULT_SCHEDULING_STRATEGY;
- String controller = request.getDeploymentControllerType() != null
- ? request.getDeploymentControllerType() : DEFAULT_DEPLOYMENT_CONTROLLER;
- if (SCHEDULING_DAEMON.equals(strategy)
- && (svc.getLaunchType() == LaunchType.FARGATE || !DEFAULT_DEPLOYMENT_CONTROLLER.equals(controller))) {
- throw new AwsException("InvalidParameterException",
- "Tasks using the Fargate launch type or the CODE_DEPLOY or EXTERNAL deployment "
- + "controller types don't support the DAEMON scheduling strategy.", 400);
- }
- svc.setSchedulingStrategy(strategy);
- svc.setDeploymentController(controller);
- svc.setAvailabilityZoneRebalancing(request.getAvailabilityZoneRebalancing() != null
- ? request.getAvailabilityZoneRebalancing() : DEFAULT_AZ_REBALANCING_ON_CREATE);
- svc.setServiceConnectConfiguration(request.getServiceConnectConfiguration());
- svc.setDeploymentConfiguration(request.getDeploymentConfiguration());
- svc.setEnableExecuteCommand(request.isEnableExecuteCommand());
- svc.setEnableECSManagedTags(request.isEnableECSManagedTags());
- // Both have documented defaults a created service reports back.
- svc.setPropagateTags(request.getPropagateTags() != null
- ? request.getPropagateTags() : PROPAGATE_TAGS_NONE);
- svc.setHealthCheckGracePeriodSeconds(request.getHealthCheckGracePeriodSeconds() != null
- ? request.getHealthCheckGracePeriodSeconds() : 0);
- validateServiceRole(request, taskDef);
- svc.setRoleArn(request.getRoleArn());
- svc.setUnparsed(request.getUnparsed());
- applyServicePlatform(svc, taskDef, request.getPlatformVersion());
- svc.setStatus("ACTIVE");
- svc.setCreatedAt(Instant.now());
- svc.setLastDeploymentAt(svc.getCreatedAt());
- svc.setDeploymentId(newDeploymentId());
- if (request.getTags() != null && !request.getTags().isEmpty()) {
- svc.setTags(new LinkedHashMap<>(request.getTags()));
- }
- services.put(key, svc);
- cluster.setActiveServicesCount(cluster.getActiveServicesCount() + 1);
- persistCluster(region, cluster);
- recordServiceDeployment(svc, taskDefinition, region);
- if (eventPublisher != null) {
- eventPublisher.emitDeploymentStateChange(svc, "SERVICE_DEPLOYMENT_STARTED",
- "ECS deployment " + svc.getDeploymentId() + " started.", region);
+ services.put(key, svc);
+ cluster.setActiveServicesCount(cluster.getActiveServicesCount() + 1);
+ persistCluster(region, cluster);
+ recordServiceDeployment(svc, taskDefinition, region);
+ if (eventPublisher != null) {
+ eventPublisher.emitDeploymentStateChange(svc, "SERVICE_DEPLOYMENT_STARTED",
+ "ECS deployment " + svc.getDeploymentId() + " started.", region);
+ }
+ LOG.infov("Created ECS service: {0} in cluster {1}", serviceName, cluster.getClusterName());
+ return svc;
}
- LOG.infov("Created ECS service: {0} in cluster {1}", serviceName, cluster.getClusterName());
- return svc;
}
public EcsServiceModel updateService(String clusterRef, String serviceName, String taskDefinition,
@@ -2704,111 +2712,113 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
boolean forceNewDeployment = request.isForceNewDeployment();
String key = serviceKey(region, cluster.getClusterName(), serviceName);
- EcsServiceModel svc = services.get(key);
- if (svc == null) {
- throw new AwsException("ServiceNotFoundException", "Service " + serviceName + " not found.", 404);
- }
- synchronized (svc) {
- if ("INACTIVE".equals(svc.getStatus())) {
- throw new AwsException("ServiceNotActiveException",
- "Service " + serviceName + " is not active.", 400);
+ synchronized (serviceLock(key)) {
+ EcsServiceModel svc = services.get(key);
+ if (svc == null) {
+ throw new AwsException("ServiceNotFoundException", "Service " + serviceName + " not found.", 404);
}
- // Settle the outgoing deployment against the service as it stands, before this request
- // changes its desired count or task definition: one whose tasks are up has finished,
- // whether or not anything read it. recordServiceDeployment stops it if it has not.
- currentServiceDeployment(svc);
- if (request.getDesiredCount() != null) {
- if (request.getDesiredCount() < 0) {
- throw new AwsException("InvalidParameterException", "desiredCount cannot be a negative number.", 400);
+ synchronized (svc) {
+ if ("INACTIVE".equals(svc.getStatus())) {
+ throw new AwsException("ServiceNotActiveException",
+ "Service " + serviceName + " is not active.", 400);
}
- svc.setDesiredCount(request.getDesiredCount());
- }
- // The members whose change starts new tasks, as UpdateService documents them: the network
- // configuration, the load balancers, the service registries, the Service Connect
- // configuration, the task definition and the platform version. The rest (desired count,
- // deployment configuration, the exec and managed-tag switches, placement, propagateTags,
- // the capacity provider strategy) are applied without rolling anything.
- boolean rollingChange = false;
- if (request.getNetworkConfiguration() != null) {
- rollingChange |= networkConfigurationChanged(svc.getNetworkConfiguration(),
- request.getNetworkConfiguration());
- svc.setNetworkConfiguration(request.getNetworkConfiguration());
- }
- if (request.getAvailabilityZoneRebalancing() != null) {
- svc.setAvailabilityZoneRebalancing(request.getAvailabilityZoneRebalancing());
- }
- if (request.getCapacityProviderStrategy() != null) {
- svc.setCapacityProviderStrategy(request.getCapacityProviderStrategy());
- svc.setLaunchType(request.getCapacityProviderStrategy().isEmpty()
- ? svc.getLaunchType()
- : placementFor(request.getCapacityProviderStrategy().getFirst().capacityProvider()).launchType());
- }
- if (request.getEnableExecuteCommand() != null) {
- svc.setEnableExecuteCommand(request.getEnableExecuteCommand());
- }
- if (request.getEnableECSManagedTags() != null) {
- svc.setEnableECSManagedTags(request.getEnableECSManagedTags());
- }
- if (request.getPropagateTags() != null) {
- svc.setPropagateTags(request.getPropagateTags());
- }
- if (request.getHealthCheckGracePeriodSeconds() != null) {
- svc.setHealthCheckGracePeriodSeconds(request.getHealthCheckGracePeriodSeconds());
- }
- if (request.getDeploymentConfiguration() != null) {
- svc.setDeploymentConfiguration(request.getDeploymentConfiguration());
- }
- if (request.getLoadBalancers() != null) {
- rollingChange |= loadBalancersChanged(svc.getLoadBalancers(), request.getLoadBalancers());
- svc.setLoadBalancers(request.getLoadBalancers());
- }
- if (request.getServiceRegistries() != null) {
- boolean registriesChanged = !request.getServiceRegistries().equals(svc.getServiceRegistries());
- rollingChange |= registriesChanged;
- svc.setServiceRegistries(request.getServiceRegistries());
- if (registriesChanged) {
- reconcileServiceDiscoveryRegistries(svc, cluster, region);
+ // Settle the outgoing deployment against the service as it stands, before this request
+ // changes its desired count or task definition: one whose tasks are up has finished,
+ // whether or not anything read it. recordServiceDeployment stops it if it has not.
+ currentServiceDeployment(svc);
+ if (request.getDesiredCount() != null) {
+ if (request.getDesiredCount() < 0) {
+ throw new AwsException("InvalidParameterException", "desiredCount cannot be a negative number.", 400);
+ }
+ svc.setDesiredCount(request.getDesiredCount());
}
- }
- if (request.getUnparsed() != null) {
- svc.setUnparsed(mergedUnparsed(svc.getUnparsed(), request.getUnparsed()));
- }
- // UpdateServiceRequest.serviceConnectConfiguration is documented as "This parameter
- // triggers a new service deployment", so a real change rolls the deployment the way a
- // task-definition change does. An omitted parameter is not a change and rolls nothing.
- Map<String, Object> serviceConnectConfiguration = request.getServiceConnectConfiguration();
- boolean serviceConnectChanged = serviceConnectConfiguration != null
- && !serviceConnectConfiguration.equals(svc.getServiceConnectConfiguration());
- if (serviceConnectConfiguration != null) {
- svc.setServiceConnectConfiguration(serviceConnectConfiguration);
- }
- boolean taskDefChanged = false;
- String platformVersionBefore = svc.getPlatformVersion();
- if (request.getTaskDefinition() != null) {
- TaskDefinition resolved = resolveLaunchableTaskDefinition(request.getTaskDefinition(), region);
- taskDefChanged = !resolved.getTaskDefinitionArn().equals(svc.getTaskDefinition());
- svc.setTaskDefinition(resolved.getTaskDefinitionArn());
- applyServicePlatform(svc, resolved, request.getPlatformVersion() != null
- ? request.getPlatformVersion() : svc.getPlatformVersion());
- } else if (request.getPlatformVersion() != null) {
- svc.setPlatformVersion(PLATFORM_VERSION_LATEST.equals(request.getPlatformVersion())
- ? DEFAULT_PLATFORM_VERSION : request.getPlatformVersion());
- }
- // UpdateServiceRequest.platformVersion is documented as "This parameter triggers a new
- // service deployment". Compared after resolution, so a request that asks for LATEST on a
- // service already running the version LATEST resolves to is not a change.
- rollingChange |= !Objects.equals(platformVersionBefore, svc.getPlatformVersion());
- if (taskDefChanged || forceNewDeployment || serviceConnectChanged || rollingChange) {
- svc.setDeploymentId(newDeploymentId());
- svc.setLastDeploymentAt(Instant.now());
- recordServiceDeployment(svc, svc.getTaskDefinition(), region);
- if (eventPublisher != null) {
- eventPublisher.emitDeploymentStateChange(svc, "SERVICE_DEPLOYMENT_STARTED",
- "ECS deployment " + svc.getDeploymentId() + " started.", region);
+ // The members whose change starts new tasks, as UpdateService documents them: the network
+ // configuration, the load balancers, the service registries, the Service Connect
+ // configuration, the task definition and the platform version. The rest (desired count,
+ // deployment configuration, the exec and managed-tag switches, placement, propagateTags,
+ // the capacity provider strategy) are applied without rolling anything.
+ boolean rollingChange = false;
+ if (request.getNetworkConfiguration() != null) {
+ rollingChange |= networkConfigurationChanged(svc.getNetworkConfiguration(),
+ request.getNetworkConfiguration());
+ svc.setNetworkConfiguration(request.getNetworkConfiguration());
+ }
+ if (request.getAvailabilityZoneRebalancing() != null) {
+ svc.setAvailabilityZoneRebalancing(request.getAvailabilityZoneRebalancing());
+ }
+ if (request.getCapacityProviderStrategy() != null) {
+ svc.setCapacityProviderStrategy(request.getCapacityProviderStrategy());
+ svc.setLaunchType(request.getCapacityProviderStrategy().isEmpty()
+ ? svc.getLaunchType()
+ : placementFor(request.getCapacityProviderStrategy().getFirst().capacityProvider()).launchType());
+ }
+ if (request.getEnableExecuteCommand() != null) {
+ svc.setEnableExecuteCommand(request.getEnableExecuteCommand());
+ }
+ if (request.getEnableECSManagedTags() != null) {
+ svc.setEnableECSManagedTags(request.getEnableECSManagedTags());
+ }
+ if (request.getPropagateTags() != null) {
+ svc.setPropagateTags(request.getPropagateTags());
}
+ if (request.getHealthCheckGracePeriodSeconds() != null) {
+ svc.setHealthCheckGracePeriodSeconds(request.getHealthCheckGracePeriodSeconds());
+ }
+ if (request.getDeploymentConfiguration() != null) {
+ svc.setDeploymentConfiguration(request.getDeploymentConfiguration());
+ }
+ if (request.getLoadBalancers() != null) {
+ rollingChange |= loadBalancersChanged(svc.getLoadBalancers(), request.getLoadBalancers());
+ svc.setLoadBalancers(request.getLoadBalancers());
+ }
+ if (request.getServiceRegistries() != null) {
+ boolean registriesChanged = !request.getServiceRegistries().equals(svc.getServiceRegistries());
+ rollingChange |= registriesChanged;
+ svc.setServiceRegistries(request.getServiceRegistries());
+ if (registriesChanged) {
+ reconcileServiceDiscoveryRegistries(svc, cluster, region);
+ }
+ }
+ if (request.getUnparsed() != null) {
+ svc.setUnparsed(mergedUnparsed(svc.getUnparsed(), request.getUnparsed()));
+ }
+ // UpdateServiceRequest.serviceConnectConfiguration is documented as "This parameter
+ // triggers a new service deployment", so a real change rolls the deployment the way a
+ // task-definition change does. An omitted parameter is not a change and rolls nothing.
+ Map<String, Object> serviceConnectConfiguration = request.getServiceConnectConfiguration();
+ boolean serviceConnectChanged = serviceConnectConfiguration != null
+ && !serviceConnectConfiguration.equals(svc.getServiceConnectConfiguration());
+ if (serviceConnectConfiguration != null) {
+ svc.setServiceConnectConfiguration(serviceConnectConfiguration);
+ }
+ boolean taskDefChanged = false;
+ String platformVersionBefore = svc.getPlatformVersion();
+ if (request.getTaskDefinition() != null) {
+ TaskDefinition resolved = resolveLaunchableTaskDefinition(request.getTaskDefinition(), region);
+ taskDefChanged = !resolved.getTaskDefinitionArn().equals(svc.getTaskDefinition());
+ svc.setTaskDefinition(resolved.getTaskDefinitionArn());
+ applyServicePlatform(svc, resolved, request.getPlatformVersion() != null
+ ? request.getPlatformVersion() : svc.getPlatformVersion());
+ } else if (request.getPlatformVersion() != null) {
+ svc.setPlatformVersion(PLATFORM_VERSION_LATEST.equals(request.getPlatformVersion())
+ ? DEFAULT_PLATFORM_VERSION : request.getPlatformVersion());
+ }
+ // UpdateServiceRequest.platformVersion is documented as "This parameter triggers a new
+ // service deployment". Compared after resolution, so a request that asks for LATEST on a
+ // service already running the version LATEST resolves to is not a change.
+ rollingChange |= !Objects.equals(platformVersionBefore, svc.getPlatformVersion());
+ if (taskDefChanged || forceNewDeployment || serviceConnectChanged || rollingChange) {
+ svc.setDeploymentId(newDeploymentId());
+ svc.setLastDeploymentAt(Instant.now());
+ recordServiceDeployment(svc, svc.getTaskDefinition(), region);
+ if (eventPublisher != null) {
+ eventPublisher.emitDeploymentStateChange(svc, "SERVICE_DEPLOYMENT_STARTED",
+ "ECS deployment " + svc.getDeploymentId() + " started.", region);
+ }
+ }
+ services.put(key, svc);
+ return svc;
}
- services.put(key, svc);
- return svc;
}
}
@@ -2946,51 +2956,65 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
return operatingSystemFamily;
}
+ private static Object[] newServiceLocks() {
+ Object[] locks = new Object[SERVICE_LOCK_STRIPES];
+ for (int i = 0; i < locks.length; i++) {
+ locks[i] = new Object();
+ }
+ return locks;
+ }
+
+ private Object serviceLock(String key) {
+ return serviceLocks[Math.floorMod(key.hashCode(), SERVICE_LOCK_STRIPES)];
+ }
+
public EcsServiceModel deleteService(String clusterRef, String serviceName, boolean force, String region) {
EcsCluster cluster = resolveClusterOrDefault(clusterRef, region);
serviceName = extractServiceName(serviceName);
String key = serviceKey(region, cluster.getClusterName(), serviceName);
- EcsServiceModel svc = services.get(key);
- if (svc == null) {
- throw new AwsException("ServiceNotFoundException", "Service " + serviceName + " not found.", 404);
- }
- if ("INACTIVE".equals(svc.getStatus())) {
+ synchronized (serviceLock(key)) {
+ EcsServiceModel svc = services.get(key);
+ if (svc == null) {
+ throw new AwsException("ServiceNotFoundException", "Service " + serviceName + " not found.", 404);
+ }
+ if ("INACTIVE".equals(svc.getStatus())) {
+ return svc;
+ }
+ if (!force && svc.getDesiredCount() > 0) {
+ throw new AwsException("InvalidParameterException",
+ "The service cannot be stopped. Update the service to 0 tasks or use the force flag.", 400);
+ }
+ svc.setStatus("INACTIVE");
+ svc.setDesiredCount(0);
+ cluster.setActiveServicesCount(Math.max(0, cluster.getActiveServicesCount() - 1));
+ services.put(key, svc);
+ persistCluster(region, cluster);
+ // Stop tasks before removing the service from the map, so the per-task
+ // ELBv2 deregistration hook can still resolve the service's loadBalancers.
+ tasks.values().stream()
+ .filter(t -> ownedBy(t, svc, cluster))
+ .filter(t -> !TaskStatus.STOPPED.name().equals(t.getLastStatus()))
+ .forEach(t -> {
+ try {
+ stopTask(cluster.getClusterName(), t.getTaskArn(), "Service deleted",
+ STOP_CODE_SERVICE_SCHEDULER_INITIATED, region);
+ } catch (Exception e) {
+ LOG.warnv("Failed to stop task {0} on service delete: {1}",
+ t.getTaskArn(), e.getMessage());
+ }
+ });
+ // AWS deletes the service's deployments and revisions with the service. A service model
+ // restored from the store may have no ARN, and then owns none.
+ String serviceArn = svc.getServiceArn();
+ if (serviceArn != null) {
+ serviceDeployments.values().removeIf(d -> serviceArn.equals(d.getServiceArn()));
+ serviceRevisions.values().removeIf(r -> serviceArn.equals(r.getServiceArn()));
+ launchFailures.remove(serviceArn);
+ }
return svc;
}
- if (!force && svc.getDesiredCount() > 0) {
- throw new AwsException("InvalidParameterException",
- "The service cannot be stopped. Update the service to 0 tasks or use the force flag.", 400);
- }
- svc.setStatus("INACTIVE");
- svc.setDesiredCount(0);
- cluster.setActiveServicesCount(Math.max(0, cluster.getActiveServicesCount() - 1));
- services.put(key, svc);
- persistCluster(region, cluster);
- // Stop tasks before removing the service from the map, so the per-task
- // ELBv2 deregistration hook can still resolve the service's loadBalancers.
- tasks.values().stream()
- .filter(t -> ownedBy(t, svc, cluster))
- .filter(t -> !TaskStatus.STOPPED.name().equals(t.getLastStatus()))
- .forEach(t -> {
- try {
- stopTask(cluster.getClusterName(), t.getTaskArn(), "Service deleted",
- STOP_CODE_SERVICE_SCHEDULER_INITIATED, region);
- } catch (Exception e) {
- LOG.warnv("Failed to stop task {0} on service delete: {1}",
- t.getTaskArn(), e.getMessage());
- }
- });
- // AWS deletes the service's deployments and revisions with the service. A service model
- // restored from the store may have no ARN, and then owns none.
- String serviceArn = svc.getServiceArn();
- if (serviceArn != null) {
- serviceDeployments.values().removeIf(d -> serviceArn.equals(d.getServiceArn()));
- serviceRevisions.values().removeIf(r -> serviceArn.equals(r.getServiceArn()));
- launchFailures.remove(serviceArn);
- }
- return svc;
}
/** The services that resolved, plus a {@code MISSING} failure for each reference that did not. */
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeleteDeploymentsTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeleteDeploymentsTest.java
index 52b26936dd..48b53814ff 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeleteDeploymentsTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDeleteDeploymentsTest.java
@@ -16,11 +16,20 @@ import org.junit.jupiter.api.Test;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import static io.github.hectorvent.floci.testing.ThreadStates.awaitBlockedOrDone;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
/**
@@ -84,6 +93,137 @@ class EcsServiceDeleteDeploymentsTest {
"the new deployment does not roll from the deleted service's revision");
}
+ @Test
+ void anUpdateThatPassedItsStatusCheckCannotRestoreRecordsAfterDelete() throws Exception {
+ EcsService service = spy(newMockModeService());
+ service.createCluster(CLUSTER, REGION);
+ registerTaskDef(service);
+ service.createService(CLUSTER, "racy", "del-fam", 0, LaunchType.FARGATE, List.of(), null, REGION);
+
+ CountDownLatch passedCheck = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicReference<Thread> updater = new AtomicReference<>();
+ doAnswer(inv -> {
+ if (Thread.currentThread() == updater.get()) {
+ passedCheck.countDown();
+ release.await();
+ }
+ return inv.callRealMethod();
+ }).when(service).currentServiceDeployment(any());
+
+ Thread update = worker(() -> service.updateService(CLUSTER, "racy", null, null, null, null, true, REGION));
+ Thread delete = worker(() -> service.deleteService(CLUSTER, "racy", true, REGION));
+ updater.set(update);
+ try {
+ update.start();
+ assertTrue(passedCheck.await(10, TimeUnit.SECONDS));
+ delete.start();
+ awaitBlockedOrDone(delete);
+ } finally {
+ release.countDown();
+ joinAll(update, delete);
+ }
+
+ assertEquals(List.of(), service.listServiceDeployments("racy", CLUSTER, null, REGION),
+ "an update racing a delete leaves no deployment behind");
+ assertEquals("INACTIVE",
+ service.describeServices(CLUSTER, List.of("racy"), REGION).getFirst().getStatus());
+ }
+
+ @Test
+ void aCreateRacingADeleteKeepsItsOwnDeployment() throws Exception {
+ EcsService service = newMockModeService();
+ service.createCluster(CLUSTER, REGION);
+ registerTaskDef(service);
+ service.createService(CLUSTER, "swap", "del-fam", 1, LaunchType.FARGATE, List.of(), null, REGION);
+ service.reconcileServices();
+
+ Thread delete = worker(() -> service.deleteService(CLUSTER, "swap", true, REGION));
+ Thread create = worker(() ->
+ service.createService(CLUSTER, "swap", "del-fam", 0, LaunchType.FARGATE, List.of(), null, REGION));
+ // Runs while the delete is between marking the service INACTIVE and cleaning up its records.
+ doAnswer(inv -> {
+ if (Thread.currentThread() == delete && create.getState() == Thread.State.NEW) {
+ create.start();
+ awaitBlockedOrDone(create);
+ }
+ return null;
+ }).when(containerManager).releaseTaskNetwork(any(), any());
+ delete.start();
+ joinAll(delete, create);
+
+ assertEquals("ACTIVE", service.describeServices(CLUSTER, List.of("swap"), REGION).getFirst().getStatus());
+ assertEquals(1, service.listServiceDeployments("swap", CLUSTER, null, REGION).size(),
+ "the recreated service keeps the deployment it recorded");
+ }
+
+ @Test
+ void aResetDuringADeleteDoesNotLetARecreateLoseItsDeployment() throws Exception {
+ EcsService service = newMockModeService();
+ service.createCluster(CLUSTER, REGION);
+ registerTaskDef(service);
+ service.createService(CLUSTER, "reset", "del-fam", 1, LaunchType.FARGATE, List.of(), null, REGION);
+ service.reconcileServices();
+
+ CountDownLatch stopping = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ Thread delete = worker(() -> service.deleteService(CLUSTER, "reset", true, REGION));
+ Thread create = worker(() ->
+ service.createService(CLUSTER, "reset", "del-fam", 0, LaunchType.FARGATE, List.of(), null, REGION));
+ // Parks the delete between marking the service INACTIVE and cleaning up its records.
+ doAnswer(inv -> {
+ if (Thread.currentThread() == delete) {
+ stopping.countDown();
+ release.await();
+ }
+ return null;
+ }).when(containerManager).releaseTaskNetwork(any(), any());
+ try {
+ delete.start();
+ assertTrue(stopping.await(10, TimeUnit.SECONDS), "delete reached task stop");
+ service.clear();
+ create.start();
+ awaitBlockedOrDone(create);
+ } finally {
+ release.countDown();
+ joinAll(delete, create);
+ }
+
+ assertEquals("ACTIVE", service.describeServices(CLUSTER, List.of("reset"), REGION).getFirst().getStatus());
+ assertEquals(1, service.listServiceDeployments("reset", CLUSTER, null, REGION).size(),
+ "a service recreated after a reset keeps its deployment when the older delete resumes");
+ }
+
+ private final ConcurrentLinkedQueue<Throwable> workerFailures = new ConcurrentLinkedQueue<>();
+
+ /** A thread that records what it throws, so a failed worker cannot leave a passing test behind. */
+ private Thread worker(Runnable body) {
+ Thread t = new Thread(() -> {
+ try {
+ body.run();
+ } catch (Throwable e) {
+ workerFailures.add(e);
+ }
+ });
+ t.setDaemon(true);
+ return t;
+ }
+
+ private void joinAll(Thread... threads) throws InterruptedException {
+ for (Thread t : threads) {
+ t.join(10_000);
+ }
+ for (Thread t : threads) {
+ if (t.isAlive()) {
+ t.interrupt();
+ }
+ }
+ for (Thread t : threads) {
+ assertTrue(!t.isAlive(), "worker thread finished");
+ }
+ assertTrue(workerFailures.isEmpty(), "workers threw: " + workerFailures);
+ }
+
private static ServiceDeployment onlyDeployment(EcsService service, String serviceName) {
List<ServiceDeployment> deployments =
service.listServiceDeploymentsDetailed(serviceName, CLUSTER, null, REGION);
@@ -98,13 +238,15 @@ class EcsServiceDeleteDeploymentsTest {
service.registerTaskDefinition("del-fam", List.of(cd), null, null, null, null, null, List.of(), REGION);
}
- private static EcsService newMockModeService() {
+ private final EcsContainerManager containerManager = mock(EcsContainerManager.class);
+
+ private EcsService newMockModeService() {
EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
when(config.services().ecs().mock()).thenReturn(true);
when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
EcsService service = new EcsService(
new RegionResolver(REGION, "000000000000"),
- mock(EcsContainerManager.class),
+ containerManager,
config,
mock(EcsLoadBalancerRegistrar.class),
new InMemoryStorageFactory(),
diff --git a/src/test/java/io/github/hectorvent/floci/testing/ThreadStates.java b/src/test/java/io/github/hectorvent/floci/testing/ThreadStates.java
index a3f50e8fef..8a632d9a03 100644
--- a/src/test/java/io/github/hectorvent/floci/testing/ThreadStates.java
+++ b/src/test/java/io/github/hectorvent/floci/testing/ThreadStates.java
@@ -3,6 +3,10 @@ package io.github.hectorvent.floci.testing;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BooleanSupplier;
+import java.util.function.Supplier;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
/** Waiting for a thread to reach a lock, for tests that force two requests to overlap. */
public final class ThreadStates {
@@ -17,13 +21,33 @@ public final class ThreadStates {
*/
public static void awaitBlockedOrDone(AtomicReference<Thread> thread, Future<?> task)
throws InterruptedException {
- long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
- while (System.nanoTime() < deadline && !task.isDone()) {
- Thread current = thread.get();
- if (current != null && current.getState() == Thread.State.BLOCKED) {
- return;
+ awaitBlockedOrDone(thread::get, task::isDone, 5);
+ }
+
+ /**
+ * Returns once {@code thread} is blocked entering a monitor or is no longer alive. Same wait as
+ * the {@link Future} overload, for a test that owns the {@link Thread} directly; gives up after
+ * ten seconds and fails, since a thread still running unblocked by then is not overlapping the
+ * way the test needs.
+ */
+ public static void awaitBlockedOrDone(Thread thread) throws InterruptedException {
+ assertTrue(awaitBlockedOrDone(() -> thread, () -> !thread.isAlive(), 10),
+ "worker blocked or finished before timeout");
+ }
+
+ private static boolean awaitBlockedOrDone(Supplier<Thread> thread, BooleanSupplier done, long timeoutSeconds)
+ throws InterruptedException {
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds);
+ while (System.nanoTime() < deadline && !done.getAsBoolean()) {
+ if (isBlocked(thread.get())) {
+ return true;
}
Thread.sleep(5);
}
+ return done.getAsBoolean() || isBlocked(thread.get());
+ }
+
+ private static boolean isBlocked(Thread thread) {
+ return thread != null && thread.getState() == Thread.State.BLOCKED;
}
}