Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ec2): release a capture when a whole batch of instances is terminated (#4127)
Allen Sanbornaba5f4e919f3
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.javaindex 2bbb141c80..0cce6871e3 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java@@ -1283,6 +1283,26 @@ public class Ec2ContainerManager {* Sets terminatedAt for TTL pruning.*/public void terminate(Instance instance) {+ terminate(instance, null);+ }++ /**+ * Terminates an instance and runs {@code afterTeardown} once the container is actually gone.+ *+ * <p>The teardown scheduled here is asynchronous, so anything that depends on the container+ * no longer holding a Docker resource cannot run at the call site. Reclaiming a deregistered+ * AMI's captured layer is the case that needs this: attempted while the container still+ * exists, the daemon refuses to delete the image, and with no later attempt the layer stays+ * on disk for the lifetime of the emulator.+ *+ * <p>The callback runs on the teardown executor, and only once the container is established+ * to be gone: removed, already absent, or never launched. Everything after removal can fail+ * without making the callback wrong, so it runs from a finally block, but running it when+ * removal itself failed would attempt work Docker is bound to refuse with no later retry+ * behind it. A skip is logged. Anything the callback throws is logged and swallowed rather+ * than killing the executor task.+ */+ public void terminate(Instance instance, Runnable afterTeardown) {String containerId;String containerIp;String imdsSourceIp;@@ -1295,39 +1315,62 @@ public class Ec2ContainerManager {instance.setState(InstanceState.shuttingDown());}executor.submit(() -> {- portForwardManager.unpublishAll(instance);- if (containerId != null) {- try {- dockerClient.removeContainerCmd(containerId).withForce(true).exec();- } catch (NotFoundException e) {- // already gone- } catch (Exception e) {- LOG.warnv("Error removing EC2 container {0}: {1}", containerId, e.getMessage());+ // The post-teardown hook's only precondition; see the Javadoc above for why.+ boolean containerGone = false;+ try {+ portForwardManager.unpublishAll(instance);+ if (containerId == null) {+ // Nothing was ever launched, so there is no container to hold the image.+ containerGone = true;}- try {- // iptables/veth teardown lags behind container removal; prevents port-reuse conflicts.- Thread.sleep(500);- } catch (InterruptedException ie) {- Thread.currentThread().interrupt();+ if (containerId != null) {+ try {+ dockerClient.removeContainerCmd(containerId).withForce(true).exec();+ containerGone = true;+ } catch (NotFoundException e) {+ // already gone+ containerGone = true;+ } catch (Exception e) {+ LOG.warnv("Error removing EC2 container {0}: {1}", containerId, e.getMessage());+ }+ try {+ // iptables/veth teardown lags behind container removal; prevents port-reuse conflicts.+ Thread.sleep(500);+ } catch (InterruptedException ie) {+ Thread.currentThread().interrupt();+ }+ }+ if (firewallManager != null && firewallManager.enabled()+ && instance.getNetworkInterfaces() != null && !instance.getNetworkInterfaces().isEmpty()) {+ firewallManager.unregister(instance.getNetworkInterfaces().getFirst().getNetworkInterfaceId());+ }+ if (sshHostPort > 0) {+ portAllocator.release(sshHostPort);+ }+ metadataServer.unregisterContainer(containerIp, instance);+ metadataServer.unregisterContainer(imdsSourceIp, instance);+ metadataServer.unregisterInstance(instance);+ // Give the address back only now that the container is gone: releasing it while+ // Docker still holds the endpoint would hand the same IP to the next launch and+ // have Docker refuse it.+ vpcNetworkManager.releasePrivateIp(instance.getRegion(), instance.getSubnetId(),+ instance.getPrivateIpAddress());+ instance.setState(InstanceState.terminated());+ instance.setTerminatedAt(System.currentTimeMillis());+ } finally {+ if (afterTeardown != null && containerGone) {+ try {+ afterTeardown.run();+ } catch (Exception e) {+ LOG.warnv("Post-teardown hook failed for instance {0}: {1}",+ instance.getInstanceId(), e.getMessage());+ }+ } else if (afterTeardown != null) {+ LOG.warnv("Skipping post-teardown hook for instance {0}: its container is "+ + "not established to be gone, so anything waiting on the container "+ + "being gone would fail", instance.getInstanceId());}}- if (firewallManager != null && firewallManager.enabled()- && instance.getNetworkInterfaces() != null && !instance.getNetworkInterfaces().isEmpty()) {- firewallManager.unregister(instance.getNetworkInterfaces().getFirst().getNetworkInterfaceId());- }- if (sshHostPort > 0) {- portAllocator.release(sshHostPort);- }- metadataServer.unregisterContainer(containerIp, instance);- metadataServer.unregisterContainer(imdsSourceIp, instance);- metadataServer.unregisterInstance(instance);- // Give the address back only now that the container is gone: releasing it while- // Docker still holds the endpoint would hand the same IP to the next launch and- // have Docker refuse it.- vpcNetworkManager.releasePrivateIp(instance.getRegion(), instance.getSubnetId(),- instance.getPrivateIpAddress());- instance.setState(InstanceState.terminated());- instance.setTerminatedAt(System.currentTimeMillis());});}diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.javaindex db669c9012..f8cf62e777 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java@@ -33,6 +33,7 @@ import org.jboss.logging.Logger;import com.fasterxml.jackson.core.type.TypeReference;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.RequestContext;+import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.AwsRegions;@@ -3123,6 +3124,14 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {public List<Map<String, String>> terminateInstances(String region, List<String> instanceIds) {ensureDefaultResources(region);List<Map<String, String>> result = new ArrayList<>();+ // Every instance named here is going away, so none of them counts as a live dependant of+ // a capture -- not the ones already processed, which linger as shutting-down while their+ // containers are torn down asynchronously, and not the ones still waiting their turn.+ Set<String> terminating = new LinkedHashSet<>(instanceIds);+ // Captured HERE, on the request thread, because the post-teardown hook below runs on the+ // teardown executor where no request context is active and every account-aware store+ // would silently fall back to the default account.+ String owner = callerAccountId();for (String id : instanceIds) {Instance inst = getRequiredInstance(region, id);@@ -3134,7 +3143,12 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {inst.setState(InstanceState.terminated());inst.setTerminatedAt(System.currentTimeMillis());} else {- containerManager.terminate(inst);+ // Also attempt the reclaim once this container is really gone. The synchronous+ // attempt below runs while teardown is still scheduled, so for a batch whose+ // containers all tear down slowly every in-loop attempt can be refused by the+ // daemon and, without this, nothing would try again.+ containerManager.terminate(inst, () ->+ RequestScopes.runAs(owner, () -> reclaimCapturesPinnedBy(region, inst, terminating)));}// Delete root volume if deleteOnTermination (matches real AWS behavior)if (inst.getRootVolumeId() != null) {@@ -3143,7 +3157,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {releaseStandaloneInterfacesOnTermination(region, inst);instances.put(key(region, id), inst);// The last instance depending on a deregistered AMI's capture has just gone away.- reclaimCapturesPinnedBy(region, inst);+ // Cheap and usually sufficient: by here the container is often already removed.+ reclaimCapturesPinnedBy(region, inst, terminating);Map<String, String> entry = new LinkedHashMap<>();entry.put("instanceId", id);entry.put("previousState", prev.getName());@@ -5510,13 +5525,13 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {** <p>Callers must hold {@link #imageRegistryLock}.*- * @param excludedInstanceId an instance not to count as a live dependant, used by the- * terminate path where the store still reads the instance as- * running while its container is being torn down; null to count- * every live instance+ * @param excludedInstanceIds the instances not to count as live dependants, used by the+ * terminate path where the store still reads every instance of the+ * request as running or shutting-down while their containers are+ * being torn down; null or empty to count every live instance* @return true when the reference was released, so the caller knows to store the change*/- private boolean reclaimCapturedImage(Image image, String excludedInstanceId) {+ private boolean reclaimCapturedImage(Image image, Set<String> excludedInstanceIds) {String captured = image.getDockerImage();if (captured == null || config.services().ec2().mock()) {return false;@@ -5530,7 +5545,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {return false;}boolean stillLaunchable = instances.scan(i -> true).stream()- .filter(i -> i.getRegion() != null && !i.getInstanceId().equals(excludedInstanceId))+ .filter(i -> i.getRegion() != null+ && (excludedInstanceIds == null || !excludedInstanceIds.contains(i.getInstanceId()))).filter(i -> i.getState() != null && !"terminated".equals(i.getState().getName())).anyMatch(i -> captured.equals(capturedImageFor(i.getRegion(), i.getImageId())));if (stillLaunchable) {@@ -5554,8 +5570,13 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {** <p>Only tombstoned AMIs are considered: while the AMI is still registered its capture is* needed for the next launch.+ *+ * <p>{@code terminating} is the whole batch of the current request, not just this instance.+ * A reclaim attempted for the first of them can be refused by the daemon while the other+ * containers still exist, and {@link #reclaimCapturedImage} only clears the reference on+ * success, so the attempt comes round again on each remaining instance of the batch.*/- private void reclaimCapturesPinnedBy(String region, Instance terminated) {+ private void reclaimCapturesPinnedBy(String region, Instance terminated, Set<String> terminating) {if (config.services().ec2().mock()) {return;}@@ -5569,7 +5590,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {.filter(img -> DEREGISTERED_STATE.equals(img.getState())).findFirst().orElse(null);- if (holder != null && reclaimCapturedImage(holder, terminated.getInstanceId())) {+ if (holder != null && reclaimCapturedImage(holder, terminating)) {registeredImages.put(key(holder.getRegion(), holder.getImageId()), holder);}}diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2CapturedImageReclaimIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2CapturedImageReclaimIntegrationTest.javaindex e700eed32b..ca43e90b8e 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2CapturedImageReclaimIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2CapturedImageReclaimIntegrationTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.ec2;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.services.ec2.model.Image;import io.github.hectorvent.floci.services.ec2.model.Instance;import io.github.hectorvent.floci.services.ec2.model.InstanceState;@@ -13,6 +14,7 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.mockito.ArgumentCaptor;+import java.util.ArrayList;import java.util.List;import java.util.Map;import java.util.UUID;@@ -89,15 +91,31 @@ class Ec2CapturedImageReclaimIntegrationTest {when(containerManager.removeCommittedImage(anyString())).thenReturn(true);}+ /**+ * Post-teardown callbacks handed to the container manager and not yet run. Production runs+ * these on the teardown executor once the container is really gone; a test that wants to+ * model teardown still being in flight leaves them here and runs them when it chooses.+ */+ private final List<Runnable> pendingTeardowns = new ArrayList<>();+@BeforeEachvoid terminationMarksInstancesTerminated() {// Outside mock mode Ec2Service delegates termination to the container manager, which is// mocked here, so without this an instance stays "running" in the store forever and any// assertion about terminated instances would be vacuous.+ pendingTeardowns.clear();doAnswer(invocation -> {invocation.<Instance>getArgument(0).setState(InstanceState.terminated());+ runTeardownCallback(invocation.getArgument(1));return null;- }).when(containerManager).terminate(any(Instance.class));+ }).when(containerManager).terminate(any(Instance.class), any());+ }++ /** Runs a post-teardown callback if one was supplied, mirroring the production finally block. */+ private void runTeardownCallback(Object callback) {+ if (callback instanceof Runnable runnable) {+ runnable.run();+ }}private Instance launch(String imageId) {@@ -261,6 +279,11 @@ class Ec2CapturedImageReclaimIntegrationTest {// clearing the reference on a transient docker failure leaks the layer permanently. The// second termination is the later attempt: it only reaches the daemon at all if the// reference survived the first failure.+ //+ // Counts: one termination reaches the daemon twice, once in the synchronous loop and+ // once from the post-teardown callback. The second termination adds only one, because+ // its synchronous attempt succeeds and clears the reference, so the callback that+ // follows finds nothing left to reclaim.String tag = "floci-ami/ami-removal-refused:latest";Image image = captureAmi("removal-refused", tag);Instance dependent = launch(image.getImageId());@@ -269,12 +292,12 @@ class Ec2CapturedImageReclaimIntegrationTest {when(containerManager.removeCommittedImage(tag)).thenReturn(false);service.terminateInstances(REGION, List.of(dependent.getInstanceId()));- verify(containerManager, times(1)).removeCommittedImage(tag);+ verify(containerManager, times(2)).removeCommittedImage(tag);when(containerManager.removeCommittedImage(tag)).thenReturn(true);service.terminateInstances(REGION, List.of(dependent.getInstanceId()));- verify(containerManager, times(2)).removeCommittedImage(tag);+ verify(containerManager, times(3)).removeCommittedImage(tag);}// ─── Termination reclaims what deregistration had to retain ───────────────@@ -308,6 +331,143 @@ class Ec2CapturedImageReclaimIntegrationTest {verify(containerManager, never()).removeCommittedImage(tag);}+ /**+ * Reproduces what production termination actually does to the store: the container manager+ * flips the instance to {@code shutting-down} synchronously and only reaches+ * {@code terminated} once its asynchronous teardown finishes. The default stub in this class+ * terminates immediately, which hides every ordering question about a batch.+ */+ private void terminationLeavesInstancesShuttingDown() {+ doAnswer(invocation -> {+ invocation.<Instance>getArgument(0).setState(InstanceState.shuttingDown());+ if (invocation.getArgument(1) instanceof Runnable callback) {+ // Teardown has not finished, so the callback is held rather than run. A test that+ // wants to model teardown completing drains pendingTeardowns itself.+ pendingTeardowns.add(callback);+ }+ return null;+ }).when(containerManager).terminate(any(Instance.class), any());+ }++ @Test+ void terminatingAWholeBatchOfDependentsReleasesTheCapture() {+ // Every instance in one TerminateInstances request goes away together, so none of them+ // can pin the capture. Counting the siblings as live leaves the layer on disk forever:+ // the reclaim runs only from termination and deregistration, deregistration is rejected+ // the second time, and nothing revisits the batch after its teardown completes.+ String tag = "floci-ami/ami-batch:latest";+ Image image = captureAmi("batch-terminated", tag);+ terminationLeavesInstancesShuttingDown();+ Instance first = launch(image.getImageId());+ Instance second = launch(image.getImageId());+ service.deregisterImage(REGION, image.getImageId(), false);+ verify(containerManager, never()).removeCommittedImage(tag);++ service.terminateInstances(REGION,+ List.of(first.getInstanceId(), second.getInstanceId()));++ verify(containerManager).removeCommittedImage(tag);+ }++ @Test+ void aBatchReclaimTheDaemonRefusesEarlyIsRetriedLaterInTheSameBatch() {+ // With the whole batch excluded the reclaim is attempted on the first instance, while the+ // other containers may still exist and Docker may refuse to delete the layer. The+ // reference is only cleared on success, so the attempt has to come round again as the+ // rest of the batch is processed.+ String tag = "floci-ami/ami-batch-retry:latest";+ Image image = captureAmi("batch-retry", tag);+ terminationLeavesInstancesShuttingDown();+ Instance first = launch(image.getImageId());+ Instance second = launch(image.getImageId());+ service.deregisterImage(REGION, image.getImageId(), false);+ when(containerManager.removeCommittedImage(tag)).thenReturn(false, true);++ service.terminateInstances(REGION,+ List.of(first.getInstanceId(), second.getInstanceId()));++ verify(containerManager, times(2)).removeCommittedImage(tag);+ }++ @Test+ void aBatchWhoseTeardownOutlastsTheLoopIsReclaimedOnceTeardownCompletes() {+ // The in-loop attempts are not enough on their own. Container teardown is asynchronous,+ // so a batch whose containers are all still present while the loop runs has every+ // attempt refused by the daemon, and the loop then ends. Without an attempt after+ // teardown the layer would stay on disk for the lifetime of the emulator.+ String tag = "floci-ami/ami-batch-slow-teardown:latest";+ Image image = captureAmi("batch-slow-teardown", tag);+ terminationLeavesInstancesShuttingDown();+ Instance first = launch(image.getImageId());+ Instance second = launch(image.getImageId());+ service.deregisterImage(REGION, image.getImageId(), false);+ // The daemon refuses while any container of the batch still exists.+ when(containerManager.removeCommittedImage(tag)).thenReturn(false);++ service.terminateInstances(REGION,+ List.of(first.getInstanceId(), second.getInstanceId()));++ // Every synchronous attempt was refused, so the layer is still held.+ verify(containerManager, times(2)).removeCommittedImage(tag);++ // Teardown completes. The containers are gone, so the daemon now accepts the removal.+ when(containerManager.removeCommittedImage(tag)).thenReturn(true);+ pendingTeardowns.forEach(Runnable::run);++ verify(containerManager, times(3)).removeCommittedImage(tag);+ }++ @Test+ void aBatchInANonDefaultAccountStillReclaimsAfterTeardown() throws Exception {+ // The hook runs on the teardown executor, which carries no request context, so every+ // account-aware store it touches falls back to the default account. For a function of+ // this shape that is silent: capturedImageFor finds no registered image under+ // 000000000000, returns null, and the reclaim returns at its first guard with nothing+ // logged -- which is indistinguishable from "this AMI was never captured".+ //+ // The other batch tests cannot catch it. They drain pendingTeardowns on the test thread,+ // where the request context IS active, so the fallback never happens. This one drains on+ // a bare thread, which is what the executor actually gives the hook.+ String tag = "floci-ami/ami-other-account:latest";+ String otherAccount = "210987654321";++ Image image = RequestScopes.callAs(otherAccount, () -> {+ when(containerManager.commitInstance(any(Instance.class), anyString())).thenReturn(tag);+ Instance source = launch(BASE_AMI);+ Image captured = service.createImage(REGION, source.getInstanceId(),+ "other-account", "captured", true);+ service.terminateInstances(REGION, List.of(source.getInstanceId()));+ return captured;+ });++ RequestScopes.runAs(otherAccount, () -> {+ terminationLeavesInstancesShuttingDown();+ Instance first = launch(image.getImageId());+ Instance second = launch(image.getImageId());+ service.deregisterImage(REGION, image.getImageId(), false);+ when(containerManager.removeCommittedImage(tag)).thenReturn(false);+ service.terminateInstances(REGION,+ List.of(first.getInstanceId(), second.getInstanceId()));+ });++ // The two synchronous attempts ran on the request thread, where the account IS active, so+ // they found the AMI and were refused by the daemon rather than by a wrong-account lookup.+ // That is exactly why this bug is invisible to the other batch tests.+ verify(containerManager, times(2)).removeCommittedImage(tag);++ // Teardown finishes on a thread with no request scope at all, as it does in production.+ when(containerManager.removeCommittedImage(tag)).thenReturn(true);+ List<Runnable> draining = List.copyOf(pendingTeardowns);+ Thread bare = new Thread(() -> draining.forEach(Runnable::run), "teardown-no-request-scope");+ bare.start();+ bare.join(30_000);++ // The third attempt is the hook's. Without the account captured on the request thread it+ // never happens: the lookup resolves under the default account, finds nothing, and the+ // reclaim returns at its first guard, leaving the layer on disk for good.+ verify(containerManager, times(3)).removeCommittedImage(tag);+ }+@Testvoid terminatingAnInstanceOfAStillRegisteredAmiKeepsTheCapture() {// The AMI can still be launched again; its capture is not garbage.diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.javaindex d1dbb59895..724612d1d1 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java@@ -58,6 +58,7 @@ import java.util.concurrent.Executors;import java.util.concurrent.Future;import java.util.concurrent.ThreadPoolExecutor;import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;import java.util.concurrent.atomic.AtomicInteger;import java.util.concurrent.atomic.AtomicReference;import java.util.function.BooleanSupplier;@@ -1783,6 +1784,41 @@ class Ec2ContainerManagerTest {awaitUntil(() -> "running".equals(instance.getState().getName()), Duration.ofSeconds(2));}+ @Test+ void theTeardownHookRunsOnceTheContainerIsGone() throws Exception {+ // The hook exists so callers can do work that is only valid after the container has+ // released its Docker resources, so it must actually fire on the ordinary path.+ LaunchHarness harness = launchHarness();+ stubDockerPing(harness.dockerClient(), false);+ Instance instance = instance("i-teardown-hook");+ harness.manager().launch(instance, "ubuntu:24.04", null, "us-west-2");+ AtomicBoolean ran = new AtomicBoolean(false);++ harness.manager().terminate(instance, () -> ran.set(true));++ awaitUntil(ran::get, Duration.ofSeconds(2));+ }++ @Test+ void theTeardownHookIsSkippedWhenTheContainerCouldNotBeRemoved() throws Exception {+ // Running it anyway would start work that depends on the container being gone while it+ // demonstrably is not, and nothing retries behind it. Skipping and logging is the+ // honest outcome.+ LaunchHarness harness = launchHarness();+ stubDockerPing(harness.dockerClient(), false);+ Instance instance = instance("i-teardown-hook-refused");+ harness.manager().launch(instance, "ubuntu:24.04", null, "us-west-2");+ instance.setDockerContainerId("c-stuck");+ when(harness.dockerClient().removeContainerCmd("c-stuck"))+ .thenThrow(new RuntimeException("daemon refused"));+ AtomicBoolean ran = new AtomicBoolean(false);++ harness.manager().terminate(instance, () -> ran.set(true));++ awaitUntil(() -> "terminated".equals(instance.getState().getName()), Duration.ofSeconds(2));+ assertFalse(ran.get(), "the hook must not run when the container is still present");+ }+@Testvoid metadataOnlyInstanceStopsStartsAndTerminatesWithoutAContainer() throws Exception {LaunchHarness harness = launchHarness();