Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ecs): serialize concurrent reconciliation passes (#6399)
Andre Kurait70ffbb0f02ad
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.javaindex 1bbbfcff01..cc68a7ab38 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java@@ -136,6 +136,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab// Replaced by afterReset() after a state reset, whose container teardown shuts this scheduler down.private volatile ScheduledExecutorService reconciler = newReconciler();private final Object reconcilerLock = new Object();+ private final Object reconciliationPassLock = new Object();// False until a previous run's task containers are confirmed gone; no task starts before.private volatile boolean leftoverContainersRemoved = true;private volatile boolean leftoverTaskVolumesRemoved;@@ -4782,25 +4783,27 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab// ── Service Reconciliation ────────────────────────────────────────────────void reconcile() {- try {- retryPreviousRunTaskVolumes();- } catch (Exception e) {- LOG.warnv(e, "ECS previous-run task volume cleanup failed: {0}", e.getMessage());- }- try {- reconcileTasks();- } catch (Exception e) {- LOG.warnv(e, "ECS task reconciliation tick failed: {0}", e.getMessage());- }- for (String accountId : reconcilableAccountIds()) {- RequestScopes.runAs(accountId, () -> {- try {- reconcileServices();- } catch (Exception e) {- LOG.warnv(e, "ECS service reconciliation tick failed for account {0}: {1}",- accountId, e.getMessage());- }- });+ synchronized (reconciliationPassLock) {+ try {+ retryPreviousRunTaskVolumes();+ } catch (Exception e) {+ LOG.warnv(e, "ECS previous-run task volume cleanup failed: {0}", e.getMessage());+ }+ try {+ reconcileTasks();+ } catch (Exception e) {+ LOG.warnv(e, "ECS task reconciliation tick failed: {0}", e.getMessage());+ }+ for (String accountId : reconcilableAccountIds()) {+ RequestScopes.runAs(accountId, () -> {+ try {+ reconcileServices();+ } catch (Exception e) {+ LOG.warnv(e, "ECS service reconciliation tick failed for account {0}: {1}",+ accountId, e.getMessage());+ }+ });+ }}}@@ -5242,16 +5245,18 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab}void reconcileServices() {- for (Map.Entry<String, EcsServiceModel> entry : services.entrySet()) {- try {- reconcileService(entry.getKey(), entry.getValue());- } catch (Exception e) {- LOG.debugv("Error reconciling ECS service {0}: {1}", entry.getKey(), e.getMessage());- }- try {- publishContainerInsights(entry.getKey(), entry.getValue());- } catch (Exception e) {- LOG.warnv(e, "Could not publish Container Insights task counts for ECS service {0}", entry.getKey());+ synchronized (reconciliationPassLock) {+ for (Map.Entry<String, EcsServiceModel> entry : services.entrySet()) {+ try {+ reconcileService(entry.getKey(), entry.getValue());+ } catch (Exception e) {+ LOG.debugv("Error reconciling ECS service {0}: {1}", entry.getKey(), e.getMessage());+ }+ try {+ publishContainerInsights(entry.getKey(), entry.getValue());+ } catch (Exception e) {+ LOG.warnv(e, "Could not publish Container Insights task counts for ECS service {0}", entry.getKey());+ }}}}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsReconciliationConcurrencyServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsReconciliationConcurrencyServiceTest.javanew file mode 100644index 0000000000..80b32e237a--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsReconciliationConcurrencyServiceTest.java@@ -0,0 +1,119 @@+package io.github.hectorvent.floci.services.ecs;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;+import io.github.hectorvent.floci.services.ecs.model.LaunchType;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;++import java.time.Duration;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.FutureTask;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;++import static org.awaitility.Awaitility.await;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.spy;+import static org.mockito.Mockito.when;++class EcsReconciliationConcurrencyServiceTest {++ private static final String REGION = "us-east-1";++ @ParameterizedTest+ @CsvSource({"true,true", "true,false", "false,true", "false,false"})+ void concurrentPassesWaitAndLaunchOnlyTheDesiredCount(boolean firstFullTick, boolean secondFullTick)+ throws Exception {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().ecs().mock()).thenReturn(true);+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ TestStorageFactory storage = new TestStorageFactory();+ EcsService service = new EcsService(new RegionResolver(REGION, "000000000000"),+ mock(EcsContainerManager.class), config, mock(EcsLoadBalancerRegistrar.class), storage, null);+ service.initializeStorage();+ ContainerDefinition container = new ContainerDefinition();+ container.setName("app");+ container.setImage("app:1");+ service.registerTaskDefinition("family", List.of(container), null, null, null,+ null, null, List.of(), REGION);+ service.createCluster("cluster", REGION);+ service.createService("cluster", "service", "family", 1, LaunchType.FARGATE, List.of(), null, REGION);++ CountDownLatch entered = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ AtomicBoolean firstRead = new AtomicBoolean(true);+ doAnswer(invocation -> {+ if (firstRead.compareAndSet(true, false)) {+ entered.countDown();+ assertTrue(release.await(10, TimeUnit.SECONDS), "test must release the first pass");+ }+ return invocation.callRealMethod();+ }).when(storage.services()).keys();++ FutureTask<Void> first = new FutureTask<>(() -> runPass(service, firstFullTick), null);+ FutureTask<Void> second = new FutureTask<>(() -> runPass(service, secondFullTick), null);+ Thread firstThread = new Thread(first, "first-ecs-pass");+ Thread secondThread = new Thread(second, "second-ecs-pass");+ firstThread.start();+ try {+ assertTrue(entered.await(5, TimeUnit.SECONDS), "first pass must reach service storage");+ secondThread.start();+ await().atMost(Duration.ofSeconds(5)).until(() ->+ secondThread.getState() == Thread.State.BLOCKED || second.isDone());+ assertEquals(Thread.State.BLOCKED, secondThread.getState(),+ "a direct pass must wait for the running pass instead of reconciling the same deficit");+ } finally {+ release.countDown();+ firstThread.join(10_000);+ secondThread.join(10_000);+ }+ first.get(1, TimeUnit.SECONDS);+ second.get(1, TimeUnit.SECONDS);+ assertEquals(1, service.listTasks("cluster", null, null, null, REGION).size());+ assertEquals(1, service.describeServices("cluster", List.of("service"), REGION)+ .getFirst().getRunningCount());+ }++ private static void runPass(EcsService service, boolean fullTick) {+ if (fullTick) {+ service.reconcile();+ } else {+ service.reconcileServices();+ }+ }++ private static final class TestStorageFactory extends StorageFactory {+ private final Map<String, AccountAwareStorageBackend<?>> stores = new HashMap<>();++ private TestStorageFactory() {+ super(null, null);+ }++ @SuppressWarnings("unchecked")+ private AccountAwareStorageBackend<EcsServiceModel> services() {+ return (AccountAwareStorageBackend<EcsServiceModel>) stores.get("ecs-services.json");+ }++ @Override+ @SuppressWarnings("unchecked")+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,+ ignored -> spy(AccountAwareStorageBackend.inMemory("000000000000")));+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceAwaitStableTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceAwaitStableTest.javaindex 2c45af5cb0..8162f2fd5b 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceAwaitStableTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceAwaitStableTest.java@@ -219,13 +219,6 @@ class EcsServiceAwaitStableTest {}return null;}).when(containerManager).attachTaskNetwork(any(), any(), anyString());- // Its launch fails only once the wait has cleared the service's launch failure and started- // the new deployment's task: when the wait's second pass stops the previous deployment's.- doAnswer(invocation -> {- releaseStale.countDown();- stalePass.join();- return null;- }).when(containerManager).releaseTaskNetwork(any(), anyString());service.createCluster("race-cluster", REGION);registerTaskDef(service, "race-fam", "app:1");EcsServiceModel svc = service.createService("race-cluster", "race-svc", "race-fam", 1,@@ -233,10 +226,20 @@ class EcsServiceAwaitStableTest {service.awaitServiceStable(svc, GENEROUS);service.updateService("race-cluster", "race-svc", null, 2, null, REGION);stalePass.start();- assertTrue(staleLaunching.await(10, TimeUnit.SECONDS));- TaskDefinition rev2 = registerTaskDef(service, "race-fam", "app:2");- EcsServiceModel updated = service.updateService("race-cluster", "race-svc",- rev2.getTaskDefinitionArn(), 1, null, REGION);+ TaskDefinition rev2;+ EcsServiceModel updated;+ try {+ assertTrue(staleLaunching.await(10, TimeUnit.SECONDS));+ rev2 = registerTaskDef(service, "race-fam", "app:2");+ updated = service.updateService("race-cluster", "race-svc",+ rev2.getTaskDefinitionArn(), 1, null, REGION);+ } finally {+ // A new pass waits for this stale pass, so its cleanup cannot release this latch.+ releaseStale.countDown();+ stalePass.join(TimeUnit.SECONDS.toMillis(10));+ }+ assertFalse(stalePass.isAlive(), "the stale pass must finish before replacement reconciliation");+ assertTrue(service.hasLaunchFailure(svc.getServiceArn()), "the stale launch did fail");service.awaitServiceStable(updated, GENEROUS);diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.javaindex de2dcba144..789101c6f7 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceSteadyStateWaiterIntegrationTest.java@@ -230,22 +230,20 @@ class EcsServiceSteadyStateWaiterIntegrationTest {+ "\"alarmNames\":[\"" + alarmName + "\"],\"enable\":true,\"rollback\":false}},"+ NETWORK + "}");+ // The intermediate STOP_REQUESTED transition is covered without a scheduled tick in+ // EcsServiceDeploymentCircuitBreakerTest. Over HTTP, assert the stable terminal state.+ ecsService.reconcileServices();ecsService.reconcileServices();Response listed = call("ListServiceDeployments", "{\"cluster\":\"" + CLUSTER- + "\",\"service\":\"waiter-alarm-svc\",\"status\":[\"STOP_REQUESTED\"]}");+ + "\",\"service\":\"waiter-alarm-svc\",\"status\":[\"STOPPED\"]}");listed.then().body("serviceDeployments", hasSize(1));String deploymentArn = listed.jsonPath().getString("serviceDeployments[0].serviceDeploymentArn");call("DescribeServiceDeployments", "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}").then()- .body("serviceDeployments[0].status", equalTo("STOP_REQUESTED"))+ .body("serviceDeployments[0].status", equalTo("STOPPED"))+ .body("serviceDeployments[0].stoppedAt", notNullValue()).body("serviceDeployments[0].statusReason", startsWith("A deployment alarm entered the ALARM state."));-- ecsService.reconcileServices();- call("DescribeServiceDeployments", "{\"serviceDeploymentArns\":[\"" + deploymentArn + "\"]}")- .then()- .body("serviceDeployments[0].status", equalTo("STOPPED"))- .body("serviceDeployments[0].stoppedAt", notNullValue());call("DescribeServices", "{\"cluster\":\"" + CLUSTER+ "\",\"services\":[\"waiter-alarm-svc\"]}").then()