Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ecs): reconcile services in every account, not just the default one (#3297)

Matan Baruch42bc0d934fc4
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index 9750437b8f..95c177ace7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -5,6 +5,8 @@ import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.ContainerTeardown;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageBackedMap;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
@@ -40,6 +42,7 @@ import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -87,6 +90,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider {
private final Map<String, EcsTaskHandle> taskHandles = new ConcurrentHashMap<>();
// region::clusterName/serviceName → EcsServiceModel
private Map<String, EcsServiceModel> services = new ConcurrentHashMap<>();
+ private AccountAwareStorageBackend<EcsServiceModel> servicesStore;
public static final String DEFAULT_SCHEDULING_STRATEGY = "REPLICA";
public static final String SCHEDULING_DAEMON = "DAEMON";
@@ -141,8 +145,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider {
new TypeReference<Map<String, TaskDefinition>>() {});
this.latestRevisions = storageBacked("ecs-latest-revisions.json",
new TypeReference<Map<String, Integer>>() {});
- this.services = storageBacked("ecs-services.json",
+ this.servicesStore = storageFactory.create("ecs", "ecs-services.json",
new TypeReference<Map<String, EcsServiceModel>>() {});
+ this.services = new StorageBackedMap<>(this.servicesStore);
this.capacityProviders = storageBacked("ecs-capacity-providers.json",
new TypeReference<Map<String, CapacityProvider>>() {});
this.attributes = storageBacked("ecs-attributes.json",
@@ -1576,18 +1581,62 @@ public class EcsService implements ContainerTeardown, ResourceProvider {
// ── Service Reconciliation ────────────────────────────────────────────────
void reconcile() {
- reconcileTasks();
- reconcileServices();
+ try {
+ reconcileTasks();
+ } catch (Exception e) {
+ LOG.warnv(e, "ECS task reconciliation tick failed: {0}", e.getMessage());
+ }
+ for (String accountId : reconcilableAccountIds()) {
+ RequestScopes.runAs(accountId, () -> {
+ try {
+ reconcileServices();
+ } catch (Exception e) {
+ LOG.warnv(e, "ECS service reconciliation tick failed for account {0}: {1}",
+ accountId, e.getMessage());
+ }
+ });
+ }
}
- private void reconcileTasks() {
- for (String taskArn : taskHandles.keySet()) {
+ private Set<String> reconcilableAccountIds() {
+ Set<String> accountIds = new LinkedHashSet<>();
+ accountIds.add(regionResolver.getAccountId());
+ if (servicesStore != null) {
try {
- reconcileTask(taskArn);
+ for (AccountAwareStorageBackend.AccountEntry<EcsServiceModel> entry
+ : servicesStore.scanAllAccountEntries(key -> true)) {
+ accountIds.add(entry.accountId());
+ }
} catch (Exception e) {
- LOG.debugv("Error reconciling ECS task {0}: {1}", taskArn, e.getMessage());
+ LOG.warnv(e, "Could not enumerate ECS service accounts: {0}", e.getMessage());
+ }
+ }
+ return accountIds;
+ }
+
+ private void reconcileTasks() {
+ for (String taskArn : taskHandles.keySet()) {
+ RequestScopes.runAs(taskAccountId(taskArn), () -> {
+ try {
+ reconcileTask(taskArn);
+ } catch (Exception e) {
+ LOG.debugv("Error reconciling ECS task {0}: {1}", taskArn, e.getMessage());
+ }
+ });
+ }
+ }
+
+ private String taskAccountId(String taskArn) {
+ try {
+ String accountId = AwsArnUtils.parse(taskArn).accountId();
+ if (accountId != null && !accountId.isBlank()) {
+ return accountId;
}
+ } catch (IllegalArgumentException e) {
+ LOG.warnv("Could not parse an account from ECS task ARN {0}, reconciling it in the default account: {1}",
+ taskArn, e.getMessage());
}
+ return regionResolver.getAccountId();
}
private void reconcileTask(String taskArn) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsReconcileTickResilienceServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsReconcileTickResilienceServiceTest.java
new file mode 100644
index 0000000000..69226cda55
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsReconcileTickResilienceServiceTest.java
@@ -0,0 +1,67 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class EcsReconcileTickResilienceServiceTest {
+
+ private static final String REGION = "us-east-1";
+
+ @Test
+ void reconcileTickSurvivesAStorageReadFailureSoLaterTicksStillRun() {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().mock()).thenReturn(false);
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+
+ EcsService service = new EcsService(
+ new RegionResolver(REGION, "000000000000"),
+ mock(EcsContainerManager.class),
+ config,
+ mock(EcsLoadBalancerRegistrar.class),
+ new FailingServicesStorageFactory(),
+ null);
+ service.initializeStorage();
+
+ assertDoesNotThrow(service::reconcile);
+ assertDoesNotThrow(service::reconcile);
+ }
+
+ private static final class FailingServicesStorageFactory extends StorageFactory {
+
+ private final Map<String, AccountAwareStorageBackend<?>> stores = new HashMap<>();
+
+ private FailingServicesStorageFactory() {
+ super(null, null);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> AccountAwareStorageBackend<V> create(String serviceName,
+ String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName, name -> {
+ if ("ecs-services.json".equals(name)) {
+ AccountAwareStorageBackend<V> failing = mock(AccountAwareStorageBackend.class);
+ when(failing.keys()).thenThrow(
+ new IllegalStateException("storage unavailable"));
+ return failing;
+ }
+ return AccountAwareStorageBackend.inMemory("000000000000");
+ });
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsReconcilerMultiAccountServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsReconcilerMultiAccountServiceTest.java
new file mode 100644
index 0000000000..53b6794173
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsReconcilerMultiAccountServiceTest.java
@@ -0,0 +1,100 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;
+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.EcsServiceModel;
+import io.github.hectorvent.floci.services.ecs.model.EcsTask;
+import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class EcsReconcilerMultiAccountServiceTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String DEFAULT_ACCOUNT = "000000000000";
+ private static final String OTHER_ACCOUNT = "000000000001";
+
+ @Test
+ void reconcilerPlacesTasksForAServiceOwnedByANonDefaultAccount() {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().mock()).thenReturn(true);
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+
+ EcsContainerManager containerManager = mock(EcsContainerManager.class);
+ when(containerManager.startTask(any(), any(), any(), anyString()))
+ .thenReturn(mock(EcsTaskHandle.class));
+
+ AccountScopedStorageFactory storageFactory = new AccountScopedStorageFactory();
+ EcsService service = new EcsService(
+ new RegionResolver(REGION, DEFAULT_ACCOUNT),
+ containerManager,
+ config,
+ mock(EcsLoadBalancerRegistrar.class),
+ storageFactory,
+ null);
+ service.initializeStorage();
+
+ ContainerDefinition cd = new ContainerDefinition();
+ cd.setName("app");
+ cd.setImage("nginx:alpine");
+ service.registerTaskDefinition("multi-acct-fam", List.of(cd), null, null, null,
+ null, null, List.of(), REGION);
+ service.createCluster("multi-acct-cluster", REGION);
+ service.createService("multi-acct-cluster", "multi-acct-svc", "multi-acct-fam", 1,
+ LaunchType.FARGATE, List.of(), null, REGION);
+
+ AccountAwareStorageBackend<EcsServiceModel> store = storageFactory.servicesStore();
+ Map<String, EcsServiceModel> byRawKey = store.scanAllAccountsWithRawKeys();
+ String rawKey = byRawKey.keySet().iterator().next();
+ EcsServiceModel stored = byRawKey.get(rawKey);
+ store.delete(rawKey);
+ String logicalKey = rawKey.contains("/") ? rawKey.substring(rawKey.indexOf('/') + 1) : rawKey;
+ store.putForAccount(OTHER_ACCOUNT, logicalKey, stored);
+
+ service.reconcile();
+
+ EcsServiceModel reconciled =
+ store.getForAccount(OTHER_ACCOUNT, logicalKey).orElseThrow();
+ assertEquals(1, reconciled.getRunningCount(),
+ "a service owned by a non-default account must be reconciled too");
+ }
+
+ private static final class AccountScopedStorageFactory extends StorageFactory {
+
+ private final Map<String, AccountAwareStorageBackend<?>> stores = new HashMap<>();
+
+ private AccountScopedStorageFactory() {
+ super(null, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ AccountAwareStorageBackend<EcsServiceModel> servicesStore() {
+ return (AccountAwareStorageBackend<EcsServiceModel>) stores.get("ecs-services.json");
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> AccountAwareStorageBackend<V> create(String serviceName,
+ String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return (AccountAwareStorageBackend<V>) stores.computeIfAbsent(fileName,
+ ignored -> AccountAwareStorageBackend.inMemory(DEFAULT_ACCOUNT));
+ }
+ }
+}
\ No newline at end of file