Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ec2): restore and stop the instances of every account (#5103)
Robin Baeckman8eddae4ed6ba
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.javaindex a9325c52b0..4784f349c7 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java@@ -811,7 +811,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabif (portForwardManager != null) {portForwardManager.setPersister(inst -> {if (inst != null && inst.getRegion() != null && inst.getInstanceId() != null) {- instances.put(key(inst.getRegion(), inst.getInstanceId()), inst);+ putInstanceForOwner(key(inst.getRegion(), inst.getInstanceId()), inst);}});}@@ -822,27 +822,10 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, ResettabrestoreVpcNetworks();int restored = 0;- for (String key : instances.keys()) {- Instance instance = instances.get(key).orElse(null);- if (!needsMetadataRegistration(instance)) {- continue;- }- try {- restoreInstanceFirewall(instance);- } catch (Exception e) {- LOG.warnv("Could not restore EC2 firewall for {0}: {1}", instance.getInstanceId(), e.getMessage());- containerManager.stopForShutdown(instance);- instance.setState(InstanceState.stopped());- instances.put(key, instance);- continue;- }- if (containerManager.restoreMetadataRegistration(instance)) {- instances.put(key, instance);+ for (AccountAwareStorageBackend.AccountEntry<Instance> entry : entriesOfEveryAccount(instances)) {+ if (needsMetadataRegistration(entry.value())+ && RequestScopes.callAs(entry.accountId(), entry.value().getRegion(), () -> restoreInstance(entry))) {restored++;- // Container is running: re-reserve host ports and recreate any missing socat sidecars.- if (portForwardManager != null) {- portForwardManager.restore(instance);- }}}if (restored > 0) {@@ -857,6 +840,32 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, ResettabcontainerManager.reconcileOrphanedContainers(this::instanceContainerStillWanted);}+ /**+ * Runs as the instance's own account, so the security groups, prefix lists and port-forward+ * writes it resolves along the way come from that account rather than the default one.+ */+ private boolean restoreInstance(AccountAwareStorageBackend.AccountEntry<Instance> entry) {+ Instance instance = entry.value();+ try {+ restoreInstanceFirewall(instance);+ } catch (Exception e) {+ LOG.warnv("Could not restore EC2 firewall for {0}: {1}", instance.getInstanceId(), e.getMessage());+ containerManager.stopForShutdown(instance);+ instance.setState(InstanceState.stopped());+ putForOwner(instances, entry, instance);+ return false;+ }+ if (!containerManager.restoreMetadataRegistration(instance)) {+ return false;+ }+ putForOwner(instances, entry, instance);+ // Container is running: re-reserve host ports and recreate any missing socat sidecars.+ if (portForwardManager != null) {+ portForwardManager.restore(instance);+ }+ return true;+ }+private void restoreAttachedVolumesOnStartup() {Map<String, Volume> allVolumes;if (volumes instanceof AccountAwareStorageBackend<?> rawAccountAware) {@@ -952,14 +961,21 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabif (vpcNetworkManager == null || !vpcNetworkManager.enabled()) {return;}- vpcNetworkManager.reconcileOrphans((region, vpcId) -> vpcs.get(key(region, vpcId)).isPresent());- for (String storageKey : vpcs.keys()) {- vpcs.get(storageKey).ifPresent(vpc ->- vpcNetworkManager.declareVpc(vpc.getRegion(), vpc.getVpcId(), vpc.getCidrBlock()));- }- for (String storageKey : subnets.keys()) {- subnets.get(storageKey).ifPresent(subnet -> vpcNetworkManager.declareSubnet(- subnet.getRegion(), subnet.getVpcId(), subnet.getSubnetId(), subnet.getCidrBlock()));+ vpcNetworkManager.reconcileOrphans((region, vpcId) -> findAnyVpcEntry(region, vpcId).isPresent());+ // declareVpc is first-wins on overlapping CIDRs, so declare in a stable order, default account first.+ List<AccountAwareStorageBackend.AccountEntry<Vpc>> restoredVpcs = new ArrayList<>(entriesOfEveryAccount(vpcs));+ restoredVpcs.sort(Comparator+ .comparing((AccountAwareStorageBackend.AccountEntry<Vpc> entry) -> !defaultAccountId.equals(entry.accountId()))+ .thenComparing(entry -> String.valueOf(entry.accountId()))+ .thenComparing(AccountAwareStorageBackend.AccountEntry::key));+ for (AccountAwareStorageBackend.AccountEntry<Vpc> entry : restoredVpcs) {+ Vpc vpc = entry.value();+ vpcNetworkManager.declareVpc(vpc.getRegion(), vpc.getVpcId(), vpc.getCidrBlock());+ }+ for (AccountAwareStorageBackend.AccountEntry<Subnet> entry : entriesOfEveryAccount(subnets)) {+ Subnet subnet = entry.value();+ vpcNetworkManager.declareSubnet(+ subnet.getRegion(), subnet.getVpcId(), subnet.getSubnetId(), subnet.getCidrBlock());}reserveRestoredPrivateIps();}@@ -979,8 +995,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabprivate void reserveRestoredPrivateIps() {int reserved = 0;int skipped = 0;- for (String storageKey : instances.keys()) {- Instance instance = instances.get(storageKey).orElse(null);+ for (AccountAwareStorageBackend.AccountEntry<Instance> entry : entriesOfEveryAccount(instances)) {+ Instance instance = entry.value();if (instance == null || instance.getSubnetId() == null|| instance.getPrivateIpAddress() == null|| instance.getState() == null@@ -1001,6 +1017,59 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab}}+ /**+ * Every stored entry, whichever account owns it. Startup restore and shutdown run outside+ * any request, where an account-aware {@code keys()} sees only the default account.+ * A pre-multi-account key is migrated to the default account it is reported under, so it is+ * listed once and later writes and deletes reach the only copy. The migration is a write on+ * read, on purpose: it is idempotent, so the restore and shutdown paths can each call this.+ */+ private static <T> List<AccountAwareStorageBackend.AccountEntry<T>> entriesOfEveryAccount(+ StorageBackend<String, T> store) {+ if (store instanceof AccountAwareStorageBackend<?> rawAccountAware) {+ @SuppressWarnings("unchecked")+ AccountAwareStorageBackend<T> accountAware = (AccountAwareStorageBackend<T>) rawAccountAware;+ Map<String, AccountAwareStorageBackend.AccountEntry<T>> unique = new LinkedHashMap<>();+ Set<String> legacy = new HashSet<>();+ for (AccountAwareStorageBackend.AccountEntry<T> scanned : accountAware.scanAllAccountEntries(key -> true)) {+ String owned = scanned.accountId() + "/" + scanned.key();+ if (unique.putIfAbsent(owned, scanned) != null+ || accountAware.getForAccount(scanned.accountId(), scanned.key()).isEmpty()) {+ legacy.add(owned);+ }+ }+ List<AccountAwareStorageBackend.AccountEntry<T>> entries = new ArrayList<>();+ for (Map.Entry<String, AccountAwareStorageBackend.AccountEntry<T>> listed : unique.entrySet()) {+ AccountAwareStorageBackend.AccountEntry<T> scanned = listed.getValue();+ if (!legacy.contains(listed.getKey())) {+ entries.add(scanned);+ continue;+ }+ // Moves the legacy copy under its account, or drops it when a scoped twin already exists.+ accountAware.getForAccountMigratingLegacyKeys(scanned.accountId(), scanned.key(), List.of(), value -> true)+ .ifPresent(value -> entries.add(+ new AccountAwareStorageBackend.AccountEntry<>(scanned.accountId(), scanned.key(), value)));+ }+ return entries;+ }+ List<AccountAwareStorageBackend.AccountEntry<T>> entries = new ArrayList<>();+ for (String key : store.keys()) {+ store.get(key).ifPresent(value -> entries.add(new AccountAwareStorageBackend.AccountEntry<>(null, key, value)));+ }+ return entries;+ }++ private static <T> void putForOwner(StorageBackend<String, T> store,+ AccountAwareStorageBackend.AccountEntry<T> entry, T value) {+ if (store instanceof AccountAwareStorageBackend<?> rawAccountAware) {+ @SuppressWarnings("unchecked")+ AccountAwareStorageBackend<T> accountAware = (AccountAwareStorageBackend<T>) rawAccountAware;+ accountAware.putForAccount(entry.accountId(), entry.key(), value);+ return;+ }+ store.put(entry.key(), value);+ }+private void declareVpcNetwork(String region, String vpcId, String cidrBlock) {if (vpcNetworkManager != null) {vpcNetworkManager.declareVpc(region, vpcId, cidrBlock);@@ -4014,8 +4083,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabif (config.services().ec2().mock()) {return;}- for (String storeKey : Set.copyOf(instances.keys())) {- Instance inst = instances.get(storeKey).orElse(null);+ for (AccountAwareStorageBackend.AccountEntry<Instance> entry : entriesOfEveryAccount(instances)) {+ Instance inst = entry.value();if (inst == null || inst.getDockerContainerId() == null|| inst.getState() == null || !"running".equals(inst.getState().getName())) {continue;@@ -4023,7 +4092,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabtry {containerManager.stopForShutdown(inst);inst.setState(InstanceState.stopped());- instances.put(storeKey, inst);+ putForOwner(instances, entry, inst);} catch (Exception e) {LOG.warnv("Failed to stop EC2 instance container {0} on shutdown: {1}",inst.getDockerContainerId(), e.getMessage());@@ -9150,6 +9219,23 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabreturn instances.get(storageKey);}+ /**+ * The port-forward persister also runs on container executor threads, outside any request,+ * where a plain put would copy another account's instance into the default account.+ */+ private void putInstanceForOwner(String storageKey, Instance instance) {+ if (instances instanceof AccountAwareStorageBackend<?> rawAccountAware) {+ @SuppressWarnings("unchecked")+ AccountAwareStorageBackend<Instance> accountAware = (AccountAwareStorageBackend<Instance>) rawAccountAware;+ Optional<AccountAwareStorageBackend.OwnedEntry<Instance>> owner = accountAware.findAnyAccountEntry(storageKey);+ // No owner means the instance was terminated and deleted meanwhile; writing it would+ // leave a copy in the default account.+ owner.ifPresent(found -> accountAware.putForAccount(found.account(), storageKey, instance));+ return;+ }+ instances.put(storageKey, instance);+ }+private Optional<AccountAwareStorageBackend.OwnedEntry<Vpc>> findAnyVpcEntry(String region, String vpcId) {if (vpcs instanceof AccountAwareStorageBackend<?> rawAccountAware) {@SuppressWarnings("unchecked")diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2RestoreEveryAccountIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2RestoreEveryAccountIntegrationTest.javanew file mode 100644index 0000000000..ae8e60baf9--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2RestoreEveryAccountIntegrationTest.java@@ -0,0 +1,190 @@+package io.github.hectorvent.floci.services.ec2;++import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.services.ec2.model.Instance;+import io.github.hectorvent.floci.services.ec2.model.InstanceState;+import io.github.hectorvent.floci.services.ec2.model.Reservation;+import io.github.hectorvent.floci.services.ec2.portforward.Ec2PortForwardManager;+import io.quarkus.test.InjectMock;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;+import jakarta.inject.Inject;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.List;+import java.util.Map;+import java.util.concurrent.atomic.AtomicReference;+import java.util.function.Consumer;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.argThat;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++/**+ * Startup restore runs from {@code @PostConstruct}, on a thread with no request, where account-aware+ * storage resolves to the default account. These tests run it the same way, on a bare thread, for+ * an instance owned by another account.+ */+@QuarkusTest+@TestProfile(Ec2RestoreEveryAccountIntegrationTest.EnforcedContainerBackedEc2.class)+class Ec2RestoreEveryAccountIntegrationTest {++ /** Container-backed EC2 with security groups enforced, so restore resolves the instance's groups. */+ public static class EnforcedContainerBackedEc2 implements QuarkusTestProfile {+ @Override+ public Map<String, String> getConfigOverrides() {+ return Map.of("floci.services.ec2.mock", "false",+ "floci.network.security-group-enforcement.enabled", "true");+ }+ }++ private static final String REGION = "us-east-1";+ private static final String OTHER_ACCOUNT = "111122223333";++ @Inject+ Ec2Service service;++ @InjectMock+ Ec2ContainerManager containerManager;++ @InjectMock+ Ec2PortForwardManager portForwardManager;++ @InjectMock+ AmiImageResolver amiImageResolver;++ private final AtomicReference<Consumer<Instance>> persister = new AtomicReference<>();++ @BeforeEach+ void stubCollaborators() {+ when(amiImageResolver.resolveImage(any())).thenReturn(ResolvedAmiImage.minimal("floci/base:latest"));+ when(containerManager.restoreMetadataRegistration(any())).thenReturn(true);+ doAnswer(invocation -> {+ persister.set(invocation.getArgument(0));+ return null;+ }).when(portForwardManager).setPersister(any());+ }++ @Test+ void restoreKeepsAnotherAccountsInstanceRunningWhenItsSecurityGroupsAreEnforced() throws Throwable {+ Instance instance = runningInstanceOf(OTHER_ACCOUNT);++ restoreOutsideAnyRequest();++ verify(containerManager).restoreMetadataRegistration(argThat(restored -> isSame(restored, instance)));+ verify(containerManager, never()).stopForShutdown(argThat(stopped -> isSame(stopped, instance)));+ assertEquals("running", RequestScopes.callAs(OTHER_ACCOUNT,+ () -> describe(instance.getInstanceId()).getFirst().getState().getName()));+ }++ @Test+ void portForwardWritesDuringRestoreStayInTheInstancesOwnAccount() throws Throwable {+ Instance instance = runningInstanceOf(OTHER_ACCOUNT);+ doAnswer(invocation -> {+ Instance restored = invocation.getArgument(0);+ if (isSame(restored, instance)) {+ restored.getPublishedPorts().put(8080, 30080);+ persister.get().accept(restored);+ }+ return null;+ }).when(portForwardManager).restore(any());++ restoreOutsideAnyRequest();++ verify(portForwardManager).restore(argThat(restored -> isSame(restored, instance)));+ assertEquals(List.of(instance.getInstanceId()), RequestScopes.callAs(OTHER_ACCOUNT,+ () -> describe(instance.getInstanceId()).stream().map(Instance::getInstanceId).toList()));+ assertFalse(RequestScopes.callAs("000000000000", () -> describe(null).stream()+ .anyMatch(listed -> listed.getInstanceId().equals(instance.getInstanceId()))));+ }++ @Test+ void shutdownWritesAnotherAccountsInstanceOnlyToItsOwnAccount() throws Throwable {+ Instance instance = runningInstanceOf(OTHER_ACCOUNT);+ restoreOutsideAnyRequest();+ // Stopping a container unpublishes its forwards, and the port-forward manager persists that.+ doAnswer(invocation -> {+ Instance stopped = invocation.getArgument(0);+ if (isSame(stopped, instance)) {+ stopped.getPublishedPorts().clear();+ persister.get().accept(stopped);+ }+ return null;+ }).when(containerManager).stopForShutdown(any());++ runOutsideAnyRequest(service::stopManagedContainers);++ verify(containerManager).stopForShutdown(argThat(stopped -> isSame(stopped, instance)));+ assertEquals("stopped", RequestScopes.callAs(OTHER_ACCOUNT,+ () -> describe(instance.getInstanceId()).getFirst().getState().getName()));+ assertFalse(RequestScopes.callAs("000000000000", () -> describe(null).stream()+ .anyMatch(listed -> listed.getInstanceId().equals(instance.getInstanceId()))));+ }++ @Test+ void portForwardWritesFromAThreadWithNoRequestStayInTheInstancesOwnAccount() throws Throwable {+ // Ec2ContainerManager stops containers on its own executor, and unpublishing a forward there+ // persists the instance with no request, and so no account, in scope.+ Instance instance = runningInstanceOf(OTHER_ACCOUNT);+ restoreOutsideAnyRequest();++ assertNotNull(persister.get(), "the service registered no port-forward persister");+ runOutsideAnyRequest(() -> persister.get().accept(instance));++ assertEquals(List.of(instance.getInstanceId()), RequestScopes.callAs(OTHER_ACCOUNT,+ () -> describe(instance.getInstanceId()).stream().map(Instance::getInstanceId).toList()));+ assertFalse(RequestScopes.callAs("000000000000", () -> describe(null).stream()+ .anyMatch(listed -> listed.getInstanceId().equals(instance.getInstanceId()))));+ }++ private Instance runningInstanceOf(String accountId) {+ return RequestScopes.callAs(accountId, () -> {+ Instance launched = service.runInstances(REGION, "ami-amazonlinux2023", "t3.micro", 1, 1,+ null, List.of(), null, null, List.of(), null, null).getInstances().getFirst();+ launched.setDockerContainerId("container-" + launched.getInstanceId());+ launched.setState(InstanceState.running());+ return launched;+ });+ }++ private void restoreOutsideAnyRequest() throws Throwable {+ runOutsideAnyRequest(service::restoreMetadataRegistrations);+ }++ /**+ * Runs work on a fresh thread, so no request (and no account) is in scope, and rethrows what+ * it threw: an exception on another thread would otherwise end only that thread.+ */+ private static void runOutsideAnyRequest(Runnable work) throws Throwable {+ AtomicReference<Throwable> failure = new AtomicReference<>();+ Thread thread = new Thread(() -> {+ try {+ work.run();+ } catch (Throwable t) {+ failure.set(t);+ }+ });+ thread.start();+ thread.join();+ if (failure.get() != null) {+ throw failure.get();+ }+ }++ private List<Instance> describe(String instanceId) {+ return service.describeInstances(REGION, instanceId == null ? List.of() : List.of(instanceId), Map.of())+ .stream().map(Reservation::getInstances).flatMap(List::stream).toList();+ }++ private static boolean isSame(Instance candidate, Instance instance) {+ return candidate != null && instance.getInstanceId().equals(candidate.getInstanceId());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.javaindex e80310edf3..bceb7acc7e 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java@@ -64,6 +64,8 @@ import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;import org.junit.jupiter.api.Test;import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.CsvSource;+import org.mockito.ArgumentCaptor;+import org.mockito.InOrder;import java.io.StringReader;import java.lang.reflect.Field;@@ -84,6 +86,8 @@ import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicReference;import java.util.function.BiConsumer;+import java.util.function.BiPredicate;+import java.util.function.Consumer;import java.util.function.Predicate;import java.util.stream.Collectors;@@ -99,6 +103,7 @@ import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.argThat;import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.inOrder;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;import static org.mockito.Mockito.spy;@@ -3572,6 +3577,180 @@ class Ec2ServiceTest {verify(vpcNetworks, never()).reservePrivateIp("us-east-1", "subnet-a", "10.0.1.11");}+ @Test+ void restoreRebuildsTheVpcNetworkPlanOfEveryAccount() {+ // Startup runs with no request account, so an account-scoped keys() sees only the default+ // account's VPCs: every other account's VPC network was never re-declared after a restart.+ AccountAwareStorageBackend<Vpc> vpcStore = AccountAwareStorageBackend.inMemory("000000000000");+ AccountAwareStorageBackend<Subnet> subnetStore = AccountAwareStorageBackend.inMemory("000000000000");+ AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");+ Vpc vpc = new Vpc();+ vpc.setRegion("us-east-1");+ vpc.setVpcId("vpc-a");+ vpc.setCidrBlock("10.20.0.0/16");+ vpcStore.putForAccount("111122223333", "us-east-1::vpc-a", vpc);+ Subnet subnet = new Subnet();+ subnet.setRegion("us-east-1");+ subnet.setVpcId("vpc-a");+ subnet.setSubnetId("subnet-a");+ subnet.setCidrBlock("10.20.1.0/24");+ subnetStore.putForAccount("111122223333", "us-east-1::subnet-a", subnet);+ instanceStore.putForAccount("111122223333", "us-east-1::i-other",+ persistedInstance("i-other", "10.20.1.10", InstanceState.running()));++ VpcNetworkManager vpcNetworks = mock(VpcNetworkManager.class);+ when(vpcNetworks.enabled()).thenReturn(true);+ Ec2Service service = new Ec2Service(mockConfig(false), mock(Ec2ContainerManager.class),+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(),+ new InMemoryStorageFactory(Map.of("ec2-vpcs.json", vpcStore, "ec2-subnets.json", subnetStore,+ "ec2-instances.json", instanceStore)), vpcNetworks);++ service.restoreMetadataRegistrations();++ @SuppressWarnings("unchecked")+ ArgumentCaptor<BiPredicate<String, String>> stillDeclared = ArgumentCaptor.forClass(BiPredicate.class);+ verify(vpcNetworks).reconcileOrphans(stillDeclared.capture());+ assertTrue(stillDeclared.getValue().test("us-east-1", "vpc-a"));+ assertFalse(stillDeclared.getValue().test("us-east-1", "vpc-gone"));+ verify(vpcNetworks).declareVpc("us-east-1", "vpc-a", "10.20.0.0/16");+ verify(vpcNetworks).declareSubnet("us-east-1", "vpc-a", "subnet-a", "10.20.1.0/24");+ verify(vpcNetworks).reservePrivateIp("us-east-1", "subnet-a", "10.20.1.10");+ }++ @Test+ void restoreDeclaresTheDefaultAccountsVpcFirstWhenCidrsOverlap() {+ // The default account sorts after one of the others by id, so only the default-first rule puts it first.+ AccountAwareStorageBackend<Vpc> vpcStore = AccountAwareStorageBackend.inMemory("555555555555");+ for (String[] owned : new String[][] {{"999999999999", "vpc-z"}, {"111122223333", "vpc-y"}, {"555555555555", "vpc-x"}}) {+ Vpc vpc = new Vpc();+ vpc.setRegion("us-east-1");+ vpc.setVpcId(owned[1]);+ vpc.setCidrBlock("10.0.0.0/16");+ vpcStore.putForAccount(owned[0], "us-east-1::" + owned[1], vpc);+ }+ VpcNetworkManager vpcNetworks = mock(VpcNetworkManager.class);+ when(vpcNetworks.enabled()).thenReturn(true);+ EmulatorConfig config = mockConfig(false);+ when(config.defaultAccountId()).thenReturn("555555555555");+ Ec2Service service = new Ec2Service(config, mock(Ec2ContainerManager.class),+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(),+ new InMemoryStorageFactory(Map.of("ec2-vpcs.json", vpcStore)), vpcNetworks);++ service.restoreMetadataRegistrations();++ InOrder declared = inOrder(vpcNetworks);+ declared.verify(vpcNetworks).declareVpc("us-east-1", "vpc-x", "10.0.0.0/16");+ declared.verify(vpcNetworks).declareVpc("us-east-1", "vpc-y", "10.0.0.0/16");+ declared.verify(vpcNetworks).declareVpc("us-east-1", "vpc-z", "10.0.0.0/16");+ }++ @Test+ void aLatePersistForAnInstanceNoAccountHoldsWritesNothing() {+ AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");+ Ec2PortForwardManager portForwardManager = mock(Ec2PortForwardManager.class);+ Ec2Service service = new Ec2Service(mockConfig(false), mock(Ec2ContainerManager.class),+ portForwardManager, mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(),+ new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)), mock(VpcNetworkManager.class));+ service.restoreMetadataRegistrations();+ ArgumentCaptor<Consumer<Instance>> persister = ArgumentCaptor.captor();+ verify(portForwardManager).setPersister(persister.capture());++ // Terminated and deleted from its account before the port-forward persister ran.+ persister.getValue().accept(persistedInstance("i-gone", "10.20.1.10", InstanceState.terminated()));++ assertTrue(instanceStore.scanAllAccountEntries(key -> true).isEmpty());+ }++ @Test+ void restoreWritesARestoredInstanceBackUnderItsOwnAccount() {+ AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");+ Instance instance = persistedInstance("i-other", "10.20.1.10", InstanceState.running());+ instance.setDockerContainerId("container-other");+ instanceStore.putForAccount("111122223333", "us-east-1::i-other", instance);+ Ec2ContainerManager containerManager = mock(Ec2ContainerManager.class);+ when(containerManager.restoreMetadataRegistration(any())).thenReturn(true);+ Ec2Service service = new Ec2Service(mockConfig(false), containerManager,+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(),+ new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)), mock(VpcNetworkManager.class));++ service.restoreMetadataRegistrations();++ verify(containerManager).restoreMetadataRegistration(argThat(restored -> "i-other".equals(restored.getInstanceId())));+ assertEquals(List.of("111122223333"), instanceStore.scanAllAccountEntries(key -> key.endsWith("i-other")).stream()+ .map(AccountAwareStorageBackend.AccountEntry::accountId).toList());+ }++ @Test+ void restoreMigratesALegacyInstanceKeyInsteadOfCopyingIt() {+ InMemoryStorage<String, Instance> raw = new InMemoryStorage<>();+ AccountAwareStorageBackend<Instance> instanceStore = new AccountAwareStorageBackend<>(raw, null, "000000000000");+ Instance instance = persistedInstance("i-legacy", "10.20.1.10", InstanceState.running());+ instance.setDockerContainerId("container-legacy");+ raw.put("us-east-1::i-legacy", instance);+ Ec2ContainerManager containerManager = mock(Ec2ContainerManager.class);+ when(containerManager.restoreMetadataRegistration(any())).thenReturn(true);+ Ec2Service service = new Ec2Service(mockConfig(false), containerManager,+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(),+ new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)), mock(VpcNetworkManager.class));++ service.restoreMetadataRegistrations();++ verify(containerManager, times(1)).restoreMetadataRegistration(any());+ assertEquals(Set.of("000000000000/us-east-1::i-legacy"), Set.copyOf(raw.keys()));+ }++ @Test+ void restoreDropsALegacyKeyWhoseScopedTwinAlreadyExists() {+ // Left behind, the legacy copy would come back the moment the scoped one is pruned.+ InMemoryStorage<String, Instance> raw = new InMemoryStorage<>();+ AccountAwareStorageBackend<Instance> instanceStore = new AccountAwareStorageBackend<>(raw, null, "000000000000");+ Instance stale = persistedInstance("i-twin", "10.20.1.10", InstanceState.running());+ stale.setDockerContainerId("container-stale");+ raw.put("us-east-1::i-twin", stale);+ Instance current = persistedInstance("i-twin", "10.20.1.10", InstanceState.running());+ current.setDockerContainerId("container-current");+ raw.put("000000000000/us-east-1::i-twin", current);+ Ec2ContainerManager containerManager = mock(Ec2ContainerManager.class);+ when(containerManager.restoreMetadataRegistration(any())).thenReturn(true);+ Ec2Service service = new Ec2Service(mockConfig(false), containerManager,+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(),+ new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)), mock(VpcNetworkManager.class));++ service.restoreMetadataRegistrations();++ verify(containerManager, times(1)).restoreMetadataRegistration(any());+ verify(containerManager).restoreMetadataRegistration(argThat(restored ->+ "container-current".equals(restored.getDockerContainerId())));+ assertEquals(Set.of("000000000000/us-east-1::i-twin"), Set.copyOf(raw.keys()));+ }++ @Test+ void shutdownStopsAnotherAccountsRunningInstanceInItsOwnPartition() {+ AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");+ Instance instance = persistedInstance("i-other", "10.20.1.10", InstanceState.running());+ instance.setDockerContainerId("container-other");+ instanceStore.putForAccount("111122223333", "us-east-1::i-other", instance);+ Ec2ContainerManager containerManager = mock(Ec2ContainerManager.class);+ Ec2Service service = new Ec2Service(mockConfig(false), containerManager,+ mock(Ec2PortForwardManager.class), mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class),+ new Ec2InstanceTypeCatalog(),+ new InMemoryStorageFactory(Map.of("ec2-instances.json", instanceStore)), mock(VpcNetworkManager.class));++ service.stopManagedContainers();++ verify(containerManager).stopForShutdown(argThat(stopped -> "i-other".equals(stopped.getInstanceId())));+ List<AccountAwareStorageBackend.AccountEntry<Instance>> stored =+ instanceStore.scanAllAccountEntries(key -> key.endsWith("i-other"));+ assertEquals(List.of("111122223333"), stored.stream().map(AccountAwareStorageBackend.AccountEntry::accountId).toList());+ assertEquals("stopped", stored.get(0).value().getState().getName());+ }+@Testvoid restoreSurvivesAnAddressItCannotReserve() {AccountAwareStorageBackend<Instance> instanceStore = AccountAwareStorageBackend.inMemory("000000000000");