Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ec2): restore and stop the instances of every account (#5103)

Robin Baeckman8eddae4ed6ba
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
index a9325c52b0..4784f349c7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
@@ -811,7 +811,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
if (portForwardManager != null) {
portForwardManager.setPersister(inst -> {
if (inst != null && inst.getRegion() != null && inst.getInstanceId() != null) {
- instances.put(key(inst.getRegion(), inst.getInstanceId()), inst);
+ putInstanceForOwner(key(inst.getRegion(), inst.getInstanceId()), inst);
}
});
}
@@ -822,27 +822,10 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
restoreVpcNetworks();
int restored = 0;
- for (String key : instances.keys()) {
- Instance instance = instances.get(key).orElse(null);
- if (!needsMetadataRegistration(instance)) {
- continue;
- }
- try {
- restoreInstanceFirewall(instance);
- } catch (Exception e) {
- LOG.warnv("Could not restore EC2 firewall for {0}: {1}", instance.getInstanceId(), e.getMessage());
- containerManager.stopForShutdown(instance);
- instance.setState(InstanceState.stopped());
- instances.put(key, instance);
- continue;
- }
- if (containerManager.restoreMetadataRegistration(instance)) {
- instances.put(key, instance);
+ for (AccountAwareStorageBackend.AccountEntry<Instance> entry : entriesOfEveryAccount(instances)) {
+ if (needsMetadataRegistration(entry.value())
+ && RequestScopes.callAs(entry.accountId(), entry.value().getRegion(), () -> restoreInstance(entry))) {
restored++;
- // Container is running: re-reserve host ports and recreate any missing socat sidecars.
- if (portForwardManager != null) {
- portForwardManager.restore(instance);
- }
}
}
if (restored > 0) {
@@ -857,6 +840,32 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
containerManager.reconcileOrphanedContainers(this::instanceContainerStillWanted);
}
+ /**
+ * Runs as the instance's own account, so the security groups, prefix lists and port-forward
+ * writes it resolves along the way come from that account rather than the default one.
+ */
+ private boolean restoreInstance(AccountAwareStorageBackend.AccountEntry<Instance> entry) {
+ Instance instance = entry.value();
+ try {
+ restoreInstanceFirewall(instance);
+ } catch (Exception e) {
+ LOG.warnv("Could not restore EC2 firewall for {0}: {1}", instance.getInstanceId(), e.getMessage());
+ containerManager.stopForShutdown(instance);
+ instance.setState(InstanceState.stopped());
+ putForOwner(instances, entry, instance);
+ return false;
+ }
+ if (!containerManager.restoreMetadataRegistration(instance)) {
+ return false;
+ }
+ putForOwner(instances, entry, instance);
+ // Container is running: re-reserve host ports and recreate any missing socat sidecars.
+ if (portForwardManager != null) {
+ portForwardManager.restore(instance);
+ }
+ return true;
+ }
+
private void restoreAttachedVolumesOnStartup() {
Map<String, Volume> allVolumes;
if (volumes instanceof AccountAwareStorageBackend<?> rawAccountAware) {
@@ -952,14 +961,21 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
if (vpcNetworkManager == null || !vpcNetworkManager.enabled()) {
return;
}
- vpcNetworkManager.reconcileOrphans((region, vpcId) -> vpcs.get(key(region, vpcId)).isPresent());
- for (String storageKey : vpcs.keys()) {
- vpcs.get(storageKey).ifPresent(vpc ->
- vpcNetworkManager.declareVpc(vpc.getRegion(), vpc.getVpcId(), vpc.getCidrBlock()));
- }
- for (String storageKey : subnets.keys()) {
- subnets.get(storageKey).ifPresent(subnet -> vpcNetworkManager.declareSubnet(
- subnet.getRegion(), subnet.getVpcId(), subnet.getSubnetId(), subnet.getCidrBlock()));
+ vpcNetworkManager.reconcileOrphans((region, vpcId) -> findAnyVpcEntry(region, vpcId).isPresent());
+ // declareVpc is first-wins on overlapping CIDRs, so declare in a stable order, default account first.
+ List<AccountAwareStorageBackend.AccountEntry<Vpc>> restoredVpcs = new ArrayList<>(entriesOfEveryAccount(vpcs));
+ restoredVpcs.sort(Comparator
+ .comparing((AccountAwareStorageBackend.AccountEntry<Vpc> entry) -> !defaultAccountId.equals(entry.accountId()))
+ .thenComparing(entry -> String.valueOf(entry.accountId()))
+ .thenComparing(AccountAwareStorageBackend.AccountEntry::key));
+ for (AccountAwareStorageBackend.AccountEntry<Vpc> entry : restoredVpcs) {
+ Vpc vpc = entry.value();
+ vpcNetworkManager.declareVpc(vpc.getRegion(), vpc.getVpcId(), vpc.getCidrBlock());
+ }
+ for (AccountAwareStorageBackend.AccountEntry<Subnet> entry : entriesOfEveryAccount(subnets)) {
+ Subnet subnet = entry.value();
+ vpcNetworkManager.declareSubnet(
+ subnet.getRegion(), subnet.getVpcId(), subnet.getSubnetId(), subnet.getCidrBlock());
}
reserveRestoredPrivateIps();
}
@@ -979,8 +995,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
private void reserveRestoredPrivateIps() {
int reserved = 0;
int skipped = 0;
- for (String storageKey : instances.keys()) {
- Instance instance = instances.get(storageKey).orElse(null);
+ for (AccountAwareStorageBackend.AccountEntry<Instance> entry : entriesOfEveryAccount(instances)) {
+ Instance instance = entry.value();
if (instance == null || instance.getSubnetId() == null
|| instance.getPrivateIpAddress() == null
|| instance.getState() == null
@@ -1001,6 +1017,59 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
}
}
+ /**
+ * Every stored entry, whichever account owns it. Startup restore and shutdown run outside
+ * any request, where an account-aware {@code keys()} sees only the default account.
+ * A pre-multi-account key is migrated to the default account it is reported under, so it is
+ * listed once and later writes and deletes reach the only copy. The migration is a write on
+ * read, on purpose: it is idempotent, so the restore and shutdown paths can each call this.
+ */
+ private static <T> List<AccountAwareStorageBackend.AccountEntry<T>> entriesOfEveryAccount(
+ StorageBackend<String, T> store) {
+ if (store instanceof AccountAwareStorageBackend<?> rawAccountAware) {
+ @SuppressWarnings("unchecked")
+ AccountAwareStorageBackend<T> accountAware = (AccountAwareStorageBackend<T>) rawAccountAware;
+ Map<String, AccountAwareStorageBackend.AccountEntry<T>> unique = new LinkedHashMap<>();
+ Set<String> legacy = new HashSet<>();
+ for (AccountAwareStorageBackend.AccountEntry<T> scanned : accountAware.scanAllAccountEntries(key -> true)) {
+ String owned = scanned.accountId() + "/" + scanned.key();
+ if (unique.putIfAbsent(owned, scanned) != null
+ || accountAware.getForAccount(scanned.accountId(), scanned.key()).isEmpty()) {
+ legacy.add(owned);
+ }
+ }
+ List<AccountAwareStorageBackend.AccountEntry<T>> entries = new ArrayList<>();
+ for (Map.Entry<String, AccountAwareStorageBackend.AccountEntry<T>> listed : unique.entrySet()) {
+ AccountAwareStorageBackend.AccountEntry<T> scanned = listed.getValue();
+ if (!legacy.contains(listed.getKey())) {
+ entries.add(scanned);
+ continue;
+ }
+ // Moves the legacy copy under its account, or drops it when a scoped twin already exists.
+ accountAware.getForAccountMigratingLegacyKeys(scanned.accountId(), scanned.key(), List.of(), value -> true)
+ .ifPresent(value -> entries.add(
+ new AccountAwareStorageBackend.AccountEntry<>(scanned.accountId(), scanned.key(), value)));
+ }
+ return entries;
+ }
+ List<AccountAwareStorageBackend.AccountEntry<T>> entries = new ArrayList<>();
+ for (String key : store.keys()) {
+ store.get(key).ifPresent(value -> entries.add(new AccountAwareStorageBackend.AccountEntry<>(null, key, value)));
+ }
+ return entries;
+ }
+
+ private static <T> void putForOwner(StorageBackend<String, T> store,
+ AccountAwareStorageBackend.AccountEntry<T> entry, T value) {
+ if (store instanceof AccountAwareStorageBackend<?> rawAccountAware) {
+ @SuppressWarnings("unchecked")
+ AccountAwareStorageBackend<T> accountAware = (AccountAwareStorageBackend<T>) rawAccountAware;
+ accountAware.putForAccount(entry.accountId(), entry.key(), value);
+ return;
+ }
+ store.put(entry.key(), value);
+ }
+
private void declareVpcNetwork(String region, String vpcId, String cidrBlock) {
if (vpcNetworkManager != null) {
vpcNetworkManager.declareVpc(region, vpcId, cidrBlock);
@@ -4014,8 +4083,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
if (config.services().ec2().mock()) {
return;
}
- for (String storeKey : Set.copyOf(instances.keys())) {
- Instance inst = instances.get(storeKey).orElse(null);
+ for (AccountAwareStorageBackend.AccountEntry<Instance> entry : entriesOfEveryAccount(instances)) {
+ Instance inst = entry.value();
if (inst == null || inst.getDockerContainerId() == null
|| inst.getState() == null || !"running".equals(inst.getState().getName())) {
continue;
@@ -4023,7 +4092,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
try {
containerManager.stopForShutdown(inst);
inst.setState(InstanceState.stopped());
- instances.put(storeKey, inst);
+ putForOwner(instances, entry, inst);
} catch (Exception e) {
LOG.warnv("Failed to stop EC2 instance container {0} on shutdown: {1}",
inst.getDockerContainerId(), e.getMessage());
@@ -9150,6 +9219,23 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
return instances.get(storageKey);
}
+ /**
+ * The port-forward persister also runs on container executor threads, outside any request,
+ * where a plain put would copy another account's instance into the default account.
+ */
+ private void putInstanceForOwner(String storageKey, Instance instance) {
+ if (instances instanceof AccountAwareStorageBackend<?> rawAccountAware) {
+ @SuppressWarnings("unchecked")
+ AccountAwareStorageBackend<Instance> accountAware = (AccountAwareStorageBackend<Instance>) rawAccountAware;
+ Optional<AccountAwareStorageBackend.OwnedEntry<Instance>> owner = accountAware.findAnyAccountEntry(storageKey);
+ // No owner means the instance was terminated and deleted meanwhile; writing it would
+ // leave a copy in the default account.
+ owner.ifPresent(found -> accountAware.putForAccount(found.account(), storageKey, instance));
+ return;
+ }
+ instances.put(storageKey, instance);
+ }
+
private Optional<AccountAwareStorageBackend.OwnedEntry<Vpc>> findAnyVpcEntry(String region, String vpcId) {
if (vpcs instanceof AccountAwareStorageBackend<?> rawAccountAware) {
@SuppressWarnings("unchecked")
diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2RestoreEveryAccountIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2RestoreEveryAccountIntegrationTest.java
new file mode 100644
index 0000000000..ae8e60baf9
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2RestoreEveryAccountIntegrationTest.java
@@ -0,0 +1,190 @@
+package io.github.hectorvent.floci.services.ec2;
+
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.services.ec2.model.Instance;
+import io.github.hectorvent.floci.services.ec2.model.InstanceState;
+import io.github.hectorvent.floci.services.ec2.model.Reservation;
+import io.github.hectorvent.floci.services.ec2.portforward.Ec2PortForwardManager;
+import io.quarkus.test.InjectMock;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Startup restore runs from {@code @PostConstruct}, on a thread with no request, where account-aware
+ * storage resolves to the default account. These tests run it the same way, on a bare thread, for
+ * an instance owned by another account.
+ */
+@QuarkusTest
+@TestProfile(Ec2RestoreEveryAccountIntegrationTest.EnforcedContainerBackedEc2.class)
+class Ec2RestoreEveryAccountIntegrationTest {
+
+ /** Container-backed EC2 with security groups enforced, so restore resolves the instance's groups. */
+ public static class EnforcedContainerBackedEc2 implements QuarkusTestProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return Map.of("floci.services.ec2.mock", "false",
+ "floci.network.security-group-enforcement.enabled", "true");
+ }
+ }
+
+ private static final String REGION = "us-east-1";
+ private static final String OTHER_ACCOUNT = "111122223333";
+
+ @Inject
+ Ec2Service service;
+
+ @InjectMock
+ Ec2ContainerManager containerManager;
+
+ @InjectMock
+ Ec2PortForwardManager portForwardManager;
+
+ @InjectMock
+ AmiImageResolver amiImageResolver;
+
+ private final AtomicReference<Consumer<Instance>> persister = new AtomicReference<>();
+
+ @BeforeEach
+ void stubCollaborators() {
+ when(amiImageResolver.resolveImage(any())).thenReturn(ResolvedAmiImage.minimal("floci/base:latest"));
+ when(containerManager.restoreMetadataRegistration(any())).thenReturn(true);
+ doAnswer(invocation -> {
+ persister.set(invocation.getArgument(0));
+ return null;
+ }).when(portForwardManager).setPersister(any());
+ }
+
+ @Test
+ void restoreKeepsAnotherAccountsInstanceRunningWhenItsSecurityGroupsAreEnforced() throws Throwable {
+ Instance instance = runningInstanceOf(OTHER_ACCOUNT);
+
+ restoreOutsideAnyRequest();
+
+ verify(containerManager).restoreMetadataRegistration(argThat(restored -> isSame(restored, instance)));
+ verify(containerManager, never()).stopForShutdown(argThat(stopped -> isSame(stopped, instance)));
+ assertEquals("running", RequestScopes.callAs(OTHER_ACCOUNT,
+ () -> describe(instance.getInstanceId()).getFirst().getState().getName()));
+ }
+
+ @Test
+ void portForwardWritesDuringRestoreStayInTheInstancesOwnAccount() throws Throwable {
+ Instance instance = runningInstanceOf(OTHER_ACCOUNT);
+ doAnswer(invocation -> {
+ Instance restored = invocation.getArgument(0);
+ if (isSame(restored, instance)) {
+ restored.getPublishedPorts().put(8080, 30080);
+ persister.get().accept(restored);
+ }
+ return null;
+ }).when(portForwardManager).restore(any());
+
+ restoreOutsideAnyRequest();
+
+ verify(portForwardManager).restore(argThat(restored -> isSame(restored, instance)));
+ assertEquals(List.of(instance.getInstanceId()), RequestScopes.callAs(OTHER_ACCOUNT,
+ () -> describe(instance.getInstanceId()).stream().map(Instance::getInstanceId).toList()));
+ assertFalse(RequestScopes.callAs("000000000000", () -> describe(null).stream()
+ .anyMatch(listed -> listed.getInstanceId().equals(instance.getInstanceId()))));
+ }
+
+ @Test
+ void shutdownWritesAnotherAccountsInstanceOnlyToItsOwnAccount() throws Throwable {
+ Instance instance = runningInstanceOf(OTHER_ACCOUNT);
+ restoreOutsideAnyRequest();
+ // Stopping a container unpublishes its forwards, and the port-forward manager persists that.
+ doAnswer(invocation -> {
+ Instance stopped = invocation.getArgument(0);
+ if (isSame(stopped, instance)) {
+ stopped.getPublishedPorts().clear();
+ persister.get().accept(stopped);
+ }
+ return null;
+ }).when(containerManager).stopForShutdown(any());
+
+ runOutsideAnyRequest(service::stopManagedContainers);
+
+ verify(containerManager).stopForShutdown(argThat(stopped -> isSame(stopped, instance)));
+ assertEquals("stopped", RequestScopes.callAs(OTHER_ACCOUNT,
+ () -> describe(instance.getInstanceId()).getFirst().getState().getName()));
+ assertFalse(RequestScopes.callAs("000000000000", () -> describe(null).stream()
+ .anyMatch(listed -> listed.getInstanceId().equals(instance.getInstanceId()))));
+ }
+
+ @Test
+ void portForwardWritesFromAThreadWithNoRequestStayInTheInstancesOwnAccount() throws Throwable {
+ // Ec2ContainerManager stops containers on its own executor, and unpublishing a forward there
+ // persists the instance with no request, and so no account, in scope.
+ Instance instance = runningInstanceOf(OTHER_ACCOUNT);
+ restoreOutsideAnyRequest();
+
+ assertNotNull(persister.get(), "the service registered no port-forward persister");
+ runOutsideAnyRequest(() -> persister.get().accept(instance));
+
+ assertEquals(List.of(instance.getInstanceId()), RequestScopes.callAs(OTHER_ACCOUNT,
+ () -> describe(instance.getInstanceId()).stream().map(Instance::getInstanceId).toList()));
+ assertFalse(RequestScopes.callAs("000000000000", () -> describe(null).stream()
+ .anyMatch(listed -> listed.getInstanceId().equals(instance.getInstanceId()))));
+ }
+
+ private Instance runningInstanceOf(String accountId) {
+ return RequestScopes.callAs(accountId, () -> {
+ Instance launched = service.runInstances(REGION, "ami-amazonlinux2023", "t3.micro", 1, 1,
+ null, List.of(), null, null, List.of(), null, null).getInstances().getFirst();
+ launched.setDockerContainerId("container-" + launched.getInstanceId());
+ launched.setState(InstanceState.running());
+ return launched;
+ });
+ }
+
+ private void restoreOutsideAnyRequest() throws Throwable {
+ runOutsideAnyRequest(service::restoreMetadataRegistrations);
+ }
+
+ /**
+ * Runs work on a fresh thread, so no request (and no account) is in scope, and rethrows what
+ * it threw: an exception on another thread would otherwise end only that thread.
+ */
+ private static void runOutsideAnyRequest(Runnable work) throws Throwable {
+ AtomicReference<Throwable> failure = new AtomicReference<>();
+ Thread thread = new Thread(() -> {
+ try {
+ work.run();
+ } catch (Throwable t) {
+ failure.set(t);
+ }
+ });
+ thread.start();
+ thread.join();
+ if (failure.get() != null) {
+ throw failure.get();
+ }
+ }
+
+ private List<Instance> describe(String instanceId) {
+ return service.describeInstances(REGION, instanceId == null ? List.of() : List.of(instanceId), Map.of())
+ .stream().map(Reservation::getInstances).flatMap(List::stream).toList();
+ }
+
+ private static boolean isSame(Instance candidate, Instance instance) {
+ return candidate != null && instance.getInstanceId().equals(candidate.getInstanceId());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java
index e80310edf3..bceb7acc7e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java
@@ -64,6 +64,8 @@ import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
import java.io.StringReader;
import java.lang.reflect.Field;
@@ -84,6 +86,8 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
+import java.util.function.BiPredicate;
+import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@@ -99,6 +103,7 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
@@ -3572,6 +3577,180 @@ class Ec2ServiceTest {
verify(vpcNetworks, never()).reservePrivateIp("us-east-1", "subnet-a", "10.0.1.11");
}
+ @Test
+ void restoreRebuildsTheVpcNetworkPlanOfEveryAccount() {
+ // Startup runs with no request account, so an account-scoped keys() sees only the default
+ // account's VPCs: every other account's VPC network was never re-declared after a restart.
+ AccountAwareStorageBackend<Vpc> vpcStore = AccountAwareStorageBackend.inMemory("000000000000");
+ AccountAwareStorageBackend<Subnet> subnetStore = AccountAwareStorageBackend.inMemory("000000000000");
+ AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
+ Vpc vpc = new Vpc();
+ vpc.setRegion("us-east-1");
+ vpc.setVpcId("vpc-a");
+ vpc.setCidrBlock("10.20.0.0/16");
+ vpcStore.putForAccount("111122223333", "us-east-1::vpc-a", vpc);
+ Subnet subnet = new Subnet();
+ subnet.setRegion("us-east-1");
+ subnet.setVpcId("vpc-a");
+ subnet.setSubnetId("subnet-a");
+ subnet.setCidrBlock("10.20.1.0/24");
+ subnetStore.putForAccount("111122223333", "us-east-1::subnet-a", subnet);
+ instanceStore.putForAccount("111122223333", "us-east-1::i-other",
+ persistedInstance("i-other", "10.20.1.10", InstanceState.running()));
+
+ VpcNetworkManager vpcNetworks = mock(VpcNetworkManager.class);
+ when(vpcNetworks.enabled()).thenReturn(true);
+ Ec2Service service = new Ec2Service(mockConfig(false), mock(Ec2ContainerManager.class),
+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(),
+ new InMemoryStorageFactory(Map.of("ec2-vpcs.json", vpcStore, "ec2-subnets.json", subnetStore,
+ "ec2-instances.json", instanceStore)), vpcNetworks);
+
+ service.restoreMetadataRegistrations();
+
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor<BiPredicate<String, String>> stillDeclared = ArgumentCaptor.forClass(BiPredicate.class);
+ verify(vpcNetworks).reconcileOrphans(stillDeclared.capture());
+ assertTrue(stillDeclared.getValue().test("us-east-1", "vpc-a"));
+ assertFalse(stillDeclared.getValue().test("us-east-1", "vpc-gone"));
+ verify(vpcNetworks).declareVpc("us-east-1", "vpc-a", "10.20.0.0/16");
+ verify(vpcNetworks).declareSubnet("us-east-1", "vpc-a", "subnet-a", "10.20.1.0/24");
+ verify(vpcNetworks).reservePrivateIp("us-east-1", "subnet-a", "10.20.1.10");
+ }
+
+ @Test
+ void restoreDeclaresTheDefaultAccountsVpcFirstWhenCidrsOverlap() {
+ // The default account sorts after one of the others by id, so only the default-first rule puts it first.
+ AccountAwareStorageBackend<Vpc> vpcStore = AccountAwareStorageBackend.inMemory("555555555555");
+ for (String[] owned : new String[][] {{"999999999999", "vpc-z"}, {"111122223333", "vpc-y"}, {"555555555555", "vpc-x"}}) {
+ Vpc vpc = new Vpc();
+ vpc.setRegion("us-east-1");
+ vpc.setVpcId(owned[1]);
+ vpc.setCidrBlock("10.0.0.0/16");
+ vpcStore.putForAccount(owned[0], "us-east-1::" + owned[1], vpc);
+ }
+ VpcNetworkManager vpcNetworks = mock(VpcNetworkManager.class);
+ when(vpcNetworks.enabled()).thenReturn(true);
+ EmulatorConfig config = mockConfig(false);
+ when(config.defaultAccountId()).thenReturn("555555555555");
+ Ec2Service service = new Ec2Service(config, mock(Ec2ContainerManager.class),
+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(),
+ new InMemoryStorageFactory(Map.of("ec2-vpcs.json", vpcStore)), vpcNetworks);
+
+ service.restoreMetadataRegistrations();
+
+ InOrder declared = inOrder(vpcNetworks);
+ declared.verify(vpcNetworks).declareVpc("us-east-1", "vpc-x", "10.0.0.0/16");
+ declared.verify(vpcNetworks).declareVpc("us-east-1", "vpc-y", "10.0.0.0/16");
+ declared.verify(vpcNetworks).declareVpc("us-east-1", "vpc-z", "10.0.0.0/16");
+ }
+
+ @Test
+ void aLatePersistForAnInstanceNoAccountHoldsWritesNothing() {
+ AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
+ Ec2PortForwardManager portForwardManager = mock(Ec2PortForwardManager.class);
+ Ec2Service service = new Ec2Service(mockConfig(false), mock(Ec2ContainerManager.class),
+ portForwardManager, mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(),
+ new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)), mock(VpcNetworkManager.class));
+ service.restoreMetadataRegistrations();
+ ArgumentCaptor<Consumer<Instance>> persister = ArgumentCaptor.captor();
+ verify(portForwardManager).setPersister(persister.capture());
+
+ // Terminated and deleted from its account before the port-forward persister ran.
+ persister.getValue().accept(persistedInstance("i-gone", "10.20.1.10", InstanceState.terminated()));
+
+ assertTrue(instanceStore.scanAllAccountEntries(key -> true).isEmpty());
+ }
+
+ @Test
+ void restoreWritesARestoredInstanceBackUnderItsOwnAccount() {
+ AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
+ Instance instance = persistedInstance("i-other", "10.20.1.10", InstanceState.running());
+ instance.setDockerContainerId("container-other");
+ instanceStore.putForAccount("111122223333", "us-east-1::i-other", instance);
+ Ec2ContainerManager containerManager = mock(Ec2ContainerManager.class);
+ when(containerManager.restoreMetadataRegistration(any())).thenReturn(true);
+ Ec2Service service = new Ec2Service(mockConfig(false), containerManager,
+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(),
+ new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)), mock(VpcNetworkManager.class));
+
+ service.restoreMetadataRegistrations();
+
+ verify(containerManager).restoreMetadataRegistration(argThat(restored -> "i-other".equals(restored.getInstanceId())));
+ assertEquals(List.of("111122223333"), instanceStore.scanAllAccountEntries(key -> key.endsWith("i-other")).stream()
+ .map(AccountAwareStorageBackend.AccountEntry::accountId).toList());
+ }
+
+ @Test
+ void restoreMigratesALegacyInstanceKeyInsteadOfCopyingIt() {
+ InMemoryStorage<String, Instance> raw = new InMemoryStorage<>();
+ AccountAwareStorageBackend<Instance> instanceStore = new AccountAwareStorageBackend<>(raw, null, "000000000000");
+ Instance instance = persistedInstance("i-legacy", "10.20.1.10", InstanceState.running());
+ instance.setDockerContainerId("container-legacy");
+ raw.put("us-east-1::i-legacy", instance);
+ Ec2ContainerManager containerManager = mock(Ec2ContainerManager.class);
+ when(containerManager.restoreMetadataRegistration(any())).thenReturn(true);
+ Ec2Service service = new Ec2Service(mockConfig(false), containerManager,
+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(),
+ new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)), mock(VpcNetworkManager.class));
+
+ service.restoreMetadataRegistrations();
+
+ verify(containerManager, times(1)).restoreMetadataRegistration(any());
+ assertEquals(Set.of("000000000000/us-east-1::i-legacy"), Set.copyOf(raw.keys()));
+ }
+
+ @Test
+ void restoreDropsALegacyKeyWhoseScopedTwinAlreadyExists() {
+ // Left behind, the legacy copy would come back the moment the scoped one is pruned.
+ InMemoryStorage<String, Instance> raw = new InMemoryStorage<>();
+ AccountAwareStorageBackend<Instance> instanceStore = new AccountAwareStorageBackend<>(raw, null, "000000000000");
+ Instance stale = persistedInstance("i-twin", "10.20.1.10", InstanceState.running());
+ stale.setDockerContainerId("container-stale");
+ raw.put("us-east-1::i-twin", stale);
+ Instance current = persistedInstance("i-twin", "10.20.1.10", InstanceState.running());
+ current.setDockerContainerId("container-current");
+ raw.put("000000000000/us-east-1::i-twin", current);
+ Ec2ContainerManager containerManager = mock(Ec2ContainerManager.class);
+ when(containerManager.restoreMetadataRegistration(any())).thenReturn(true);
+ Ec2Service service = new Ec2Service(mockConfig(false), containerManager,
+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(),
+ new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)), mock(VpcNetworkManager.class));
+
+ service.restoreMetadataRegistrations();
+
+ verify(containerManager, times(1)).restoreMetadataRegistration(any());
+ verify(containerManager).restoreMetadataRegistration(argThat(restored ->
+ "container-current".equals(restored.getDockerContainerId())));
+ assertEquals(Set.of("000000000000/us-east-1::i-twin"), Set.copyOf(raw.keys()));
+ }
+
+ @Test
+ void shutdownStopsAnotherAccountsRunningInstanceInItsOwnPartition() {
+ AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");
+ Instance instance = persistedInstance("i-other", "10.20.1.10", InstanceState.running());
+ instance.setDockerContainerId("container-other");
+ instanceStore.putForAccount("111122223333", "us-east-1::i-other", instance);
+ Ec2ContainerManager containerManager = mock(Ec2ContainerManager.class);
+ Ec2Service service = new Ec2Service(mockConfig(false), containerManager,
+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),
+ new Ec2InstanceTypeCatalog(),
+ new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)), mock(VpcNetworkManager.class));
+
+ service.stopManagedContainers();
+
+ verify(containerManager).stopForShutdown(argThat(stopped -> "i-other".equals(stopped.getInstanceId())));
+ List<AccountAwareStorageBackend.AccountEntry<Instance>> stored =
+ instanceStore.scanAllAccountEntries(key -> key.endsWith("i-other"));
+ assertEquals(List.of("111122223333"), stored.stream().map(AccountAwareStorageBackend.AccountEntry::accountId).toList());
+ assertEquals("stopped", stored.get(0).value().getState().getName());
+ }
+
@Test
void restoreSurvivesAnAddressItCannotReserve() {
AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");