Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ecs): serialize concurrent reconciliation passes (#6399)

Andre Kurait70ffbb0f02ad
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index 1bbbfcff01..cc68a7ab38 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -136,6 +136,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
// Replaced by afterReset() after a state reset, whose container teardown shuts this scheduler down.
private volatile ScheduledExecutorService reconciler = newReconciler();
private final Object reconcilerLock = new Object();
+ private final Object reconciliationPassLock = new Object();
// False until a previous run's task containers are confirmed gone; no task starts before.
private volatile boolean leftoverContainersRemoved = true;
private volatile boolean leftoverTaskVolumesRemoved;
@@ -4782,25 +4783,27 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
// ── Service Reconciliation ────────────────────────────────────────────────
void reconcile() {
- try {
- retryPreviousRunTaskVolumes();
- } catch (Exception e) {
- LOG.warnv(e, "ECS previous-run task volume cleanup failed: {0}", e.getMessage());
- }
- try {
- reconcileTasks();
- } catch (Exception e) {
- LOG.warnv(e, "ECS task reconciliation tick failed: {0}", e.getMessage());
- }
- for (String accountId : reconcilableAccountIds()) {
- RequestScopes.runAs(accountId, () -> {
- try {
- reconcileServices();
- } catch (Exception e) {
- LOG.warnv(e, "ECS service reconciliation tick failed for account {0}: {1}",
- accountId, e.getMessage());
- }
- });
+ synchronized (reconciliationPassLock) {
+ try {
+ retryPreviousRunTaskVolumes();
+ } catch (Exception e) {
+ LOG.warnv(e, "ECS previous-run task volume cleanup failed: {0}", e.getMessage());
+ }
+ try {
+ reconcileTasks();
+ } catch (Exception e) {
+ LOG.warnv(e, "ECS task reconciliation tick failed: {0}", e.getMessage());
+ }
+ for (String accountId : reconcilableAccountIds()) {
+ RequestScopes.runAs(accountId, () -> {
+ try {
+ reconcileServices();
+ } catch (Exception e) {
+ LOG.warnv(e, "ECS service reconciliation tick failed for account {0}: {1}",
+ accountId, e.getMessage());
+ }
+ });
+ }
}
}
@@ -5242,16 +5245,18 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
}
void reconcileServices() {
- for (Map.Entry<String, EcsServiceModel> entry : services.entrySet()) {
- try {
- reconcileService(entry.getKey(), entry.getValue());
- } catch (Exception e) {
- LOG.debugv("Error reconciling ECS service {0}: {1}", entry.getKey(), e.getMessage());
- }
- try {
- publishContainerInsights(entry.getKey(), entry.getValue());
- } catch (Exception e) {
- LOG.warnv(e, "Could not publish Container Insights task counts for ECS service {0}", entry.getKey());
+ synchronized (reconciliationPassLock) {
+ for (Map.Entry<String, EcsServiceModel> entry : services.entrySet()) {
+ try {
+ reconcileService(entry.getKey(), entry.getValue());
+ } catch (Exception e) {
+ LOG.debugv("Error reconciling ECS service {0}: {1}", entry.getKey(), e.getMessage());
+ }
+ try {
+ publishContainerInsights(entry.getKey(), entry.getValue());
+ } catch (Exception e) {
+ LOG.warnv(e, "Could not publish Container Insights task counts for ECS service {0}", entry.getKey());
+ }
}
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsReconciliationConcurrencyServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsReconciliationConcurrencyServiceTest.java
new file mode 100644
index 0000000000..80b32e237a
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsReconciliationConcurrencyServiceTest.java
@@ -0,0 +1,119 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;
+import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+class EcsReconciliationConcurrencyServiceTest {
+
+ private static final String REGION = "us-east-1";
+
+ @ParameterizedTest
+ @CsvSource({"true,true", "true,false", "false,true", "false,false"})
+ void concurrentPassesWaitAndLaunchOnlyTheDesiredCount(boolean firstFullTick, boolean secondFullTick)
+ throws Exception {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().mock()).thenReturn(true);
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ TestStorageFactory storage = new TestStorageFactory();
+ EcsService service = new EcsService(new RegionResolver(REGION, "000000000000"),
+ mock(EcsContainerManager.class), config, mock(EcsLoadBalancerRegistrar.class), storage, null);
+ service.initializeStorage();
+ ContainerDefinition container = new ContainerDefinition();
+ container.setName("app");
+ container.setImage("app:1");
+ service.registerTaskDefinition("family", List.of(container), null, null, null,
+ null, null, List.of(), REGION);
+ service.createCluster("cluster", REGION);
+ service.createService("cluster", "service", "family", 1, LaunchType.FARGATE, List.of(), null, REGION);
+
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicBoolean firstRead = new AtomicBoolean(true);
+ doAnswer(invocation -> {
+ if (firstRead.compareAndSet(true, false)) {
+ entered.countDown();
+ assertTrue(release.await(10, TimeUnit.SECONDS), "test must release the first pass");
+ }
+ return invocation.callRealMethod();
+ }).when(storage.services()).keys();
+
+ FutureTask<Void> first = new FutureTask<>(() -> runPass(service, firstFullTick), null);
+ FutureTask<Void> second = new FutureTask<>(() -> runPass(service, secondFullTick), null);
+ Thread firstThread = new Thread(first, "first-ecs-pass");
+ Thread secondThread = new Thread(second, "second-ecs-pass");
+ firstThread.start();
+ try {
+ assertTrue(entered.await(5, TimeUnit.SECONDS), "first pass must reach service storage");
+ secondThread.start();
+ await().atMost(Duration.ofSeconds(5)).until(() ->
+ secondThread.getState() == Thread.State.BLOCKED || second.isDone());
+ assertEquals(Thread.State.BLOCKED, secondThread.getState(),
+ "a direct pass must wait for the running pass instead of reconciling the same deficit");
+ } finally {
+ release.countDown();
+ firstThread.join(10_000);
+ secondThread.join(10_000);
+ }
+ first.get(1, TimeUnit.SECONDS);
+ second.get(1, TimeUnit.SECONDS);
+ assertEquals(1, service.listTasks("cluster", null, null, null, REGION).size());
+ assertEquals(1, service.describeServices("cluster", List.of("service"), REGION)
+ .getFirst().getRunningCount());
+ }
+
+ private static void runPass(EcsService service, boolean fullTick) {
+ if (fullTick) {
+ service.reconcile();
+ } else {
+ service.reconcileServices();
+ }
+ }
+
+ private static final class TestStorageFactory extends StorageFactory {
+ private final Map<String, AccountAwareStorageBackend<?>> stores = new HashMap<>();
+
+ private TestStorageFactory() {
+ super(null, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ private AccountAwareStorageBackend<EcsServiceModel> services() {
+ return (AccountAwareStorageBackend<EcsServiceModel>) stores.get("ecs-services.json");
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,
+ ignored -> spy(AccountAwareStorageBackend.inMemory("000000000000")));
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceAwaitStableTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceAwaitStableTest.java
index 2c45af5cb0..8162f2fd5b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceAwaitStableTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceAwaitStableTest.java
@@ -219,13 +219,6 @@ class EcsServiceAwaitStableTest {
}
return null;
}).when(containerManager).attachTaskNetwork(any(), any(), anyString());
- // Its launch fails only once the wait has cleared the service's launch failure and started
- // the new deployment's task: when the wait's second pass stops the previous deployment's.
- doAnswer(invocation -> {
- releaseStale.countDown();
- stalePass.join();
- return null;
- }).when(containerManager).releaseTaskNetwork(any(), anyString());
service.createCluster("race-cluster", REGION);
registerTaskDef(service, "race-fam", "app:1");
EcsServiceModel svc = service.createService("race-cluster", "race-svc", "race-fam", 1,
@@ -233,10 +226,20 @@ class EcsServiceAwaitStableTest {
service.awaitServiceStable(svc, GENEROUS);
service.updateService("race-cluster", "race-svc", null, 2, null, REGION);
stalePass.start();
- assertTrue(staleLaunching.await(10, TimeUnit.SECONDS));
- TaskDefinition rev2 = registerTaskDef(service, "race-fam", "app:2");
- EcsServiceModel updated = service.updateService("race-cluster", "race-svc",
- rev2.getTaskDefinitionArn(), 1, null, REGION);
+ TaskDefinition rev2;
+ EcsServiceModel updated;
+ try {
+ assertTrue(staleLaunching.await(10, TimeUnit.SECONDS));
+ rev2 = registerTaskDef(service, "race-fam", "app:2");
+ updated = service.updateService("race-cluster", "race-svc",
+ rev2.getTaskDefinitionArn(), 1, null, REGION);
+ } finally {
+ // A new pass waits for this stale pass, so its cleanup cannot release this latch.
+ releaseStale.countDown();
+ stalePass.join(TimeUnit.SECONDS.toMillis(10));
+ }
+ assertFalse(stalePass.isAlive(), "the stale pass must finish before replacement reconciliation");
+ assertTrue(service.hasLaunchFailure(svc.getServiceArn()), "the stale launch did fail");
service.awaitServiceStable(updated, GENEROUS);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java
index de2dcba144..789101c6f7 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java
@@ -230,22 +230,20 @@ class EcsServiceSteadyStateWaiterIntegrationTest {
+ "\"alarmNames\":[\"" + alarmName + "\"],\"enable\":true,\"rollback\":false}},"
+ NETWORK + "}");
+ // The intermediate STOP_REQUESTED transition is covered without a scheduled tick in
+ // EcsServiceDeploymentCircuitBreakerTest. Over HTTP, assert the stable terminal state.
+ ecsService.reconcileServices();
ecsService.reconcileServices();
Response listed = call("ListServiceDeployments", "{\"cluster\":\"" + CLUSTER
- + "\",\"service\":\"waiter-alarm-svc\",\"status\":[\"STOP_REQUESTED\"]}");
+ + "\",\"service\":\"waiter-alarm-svc\",\"status\":[\"STOPPED\"]}");
listed.then().body("serviceDeployments", hasSize(1));
String deploymentArn = listed.jsonPath().getString("serviceDeployments[0].serviceDeploymentArn");
call("DescribeServiceDeployments", "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}")
.then()
- .body("serviceDeployments[0].status", equalTo("STOP_REQUESTED"))
+ .body("serviceDeployments[0].status", equalTo("STOPPED"))
+ .body("serviceDeployments[0].stoppedAt", notNullValue())
.body("serviceDeployments[0].statusReason", startsWith(
"A deployment alarm entered the ALARM state."));
-
- ecsService.reconcileServices();
- call("DescribeServiceDeployments", "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}")
- .then()
- .body("serviceDeployments[0].status", equalTo("STOPPED"))
- .body("serviceDeployments[0].stoppedAt", notNullValue());
call("DescribeServices", "{\"cluster\":\"" + CLUSTER
+ "\",\"services\":[\"waiter-alarm-svc\"]}")
.then()