Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ec2): release a capture when a whole batch of instances is terminated (#4127)

Allen Sanbornaba5f4e919f3
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java
index 2bbb141c80..0cce6871e3 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java
@@ -1283,6 +1283,26 @@ public class Ec2ContainerManager {
* Sets terminatedAt for TTL pruning.
*/
public void terminate(Instance instance) {
+ terminate(instance, null);
+ }
+
+ /**
+ * Terminates an instance and runs {@code afterTeardown} once the container is actually gone.
+ *
+ * <p>The teardown scheduled here is asynchronous, so anything that depends on the container
+ * no longer holding a Docker resource cannot run at the call site. Reclaiming a deregistered
+ * AMI's captured layer is the case that needs this: attempted while the container still
+ * exists, the daemon refuses to delete the image, and with no later attempt the layer stays
+ * on disk for the lifetime of the emulator.
+ *
+ * <p>The callback runs on the teardown executor, and only once the container is established
+ * to be gone: removed, already absent, or never launched. Everything after removal can fail
+ * without making the callback wrong, so it runs from a finally block, but running it when
+ * removal itself failed would attempt work Docker is bound to refuse with no later retry
+ * behind it. A skip is logged. Anything the callback throws is logged and swallowed rather
+ * than killing the executor task.
+ */
+ public void terminate(Instance instance, Runnable afterTeardown) {
String containerId;
String containerIp;
String imdsSourceIp;
@@ -1295,39 +1315,62 @@ public class Ec2ContainerManager {
instance.setState(InstanceState.shuttingDown());
}
executor.submit(() -> {
- portForwardManager.unpublishAll(instance);
- if (containerId != null) {
- try {
- dockerClient.removeContainerCmd(containerId).withForce(true).exec();
- } catch (NotFoundException e) {
- // already gone
- } catch (Exception e) {
- LOG.warnv("Error removing EC2 container {0}: {1}", containerId, e.getMessage());
+ // The post-teardown hook's only precondition; see the Javadoc above for why.
+ boolean containerGone = false;
+ try {
+ portForwardManager.unpublishAll(instance);
+ if (containerId == null) {
+ // Nothing was ever launched, so there is no container to hold the image.
+ containerGone = true;
}
- try {
- // iptables/veth teardown lags behind container removal; prevents port-reuse conflicts.
- Thread.sleep(500);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
+ if (containerId != null) {
+ try {
+ dockerClient.removeContainerCmd(containerId).withForce(true).exec();
+ containerGone = true;
+ } catch (NotFoundException e) {
+ // already gone
+ containerGone = true;
+ } catch (Exception e) {
+ LOG.warnv("Error removing EC2 container {0}: {1}", containerId, e.getMessage());
+ }
+ try {
+ // iptables/veth teardown lags behind container removal; prevents port-reuse conflicts.
+ Thread.sleep(500);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ if (firewallManager != null && firewallManager.enabled()
+ && instance.getNetworkInterfaces() != null && !instance.getNetworkInterfaces().isEmpty()) {
+ firewallManager.unregister(instance.getNetworkInterfaces().getFirst().getNetworkInterfaceId());
+ }
+ if (sshHostPort > 0) {
+ portAllocator.release(sshHostPort);
+ }
+ metadataServer.unregisterContainer(containerIp, instance);
+ metadataServer.unregisterContainer(imdsSourceIp, instance);
+ metadataServer.unregisterInstance(instance);
+ // Give the address back only now that the container is gone: releasing it while
+ // Docker still holds the endpoint would hand the same IP to the next launch and
+ // have Docker refuse it.
+ vpcNetworkManager.releasePrivateIp(instance.getRegion(), instance.getSubnetId(),
+ instance.getPrivateIpAddress());
+ instance.setState(InstanceState.terminated());
+ instance.setTerminatedAt(System.currentTimeMillis());
+ } finally {
+ if (afterTeardown != null && containerGone) {
+ try {
+ afterTeardown.run();
+ } catch (Exception e) {
+ LOG.warnv("Post-teardown hook failed for instance {0}: {1}",
+ instance.getInstanceId(), e.getMessage());
+ }
+ } else if (afterTeardown != null) {
+ LOG.warnv("Skipping post-teardown hook for instance {0}: its container is "
+ + "not established to be gone, so anything waiting on the container "
+ + "being gone would fail", instance.getInstanceId());
}
}
- if (firewallManager != null && firewallManager.enabled()
- && instance.getNetworkInterfaces() != null && !instance.getNetworkInterfaces().isEmpty()) {
- firewallManager.unregister(instance.getNetworkInterfaces().getFirst().getNetworkInterfaceId());
- }
- if (sshHostPort > 0) {
- portAllocator.release(sshHostPort);
- }
- metadataServer.unregisterContainer(containerIp, instance);
- metadataServer.unregisterContainer(imdsSourceIp, instance);
- metadataServer.unregisterInstance(instance);
- // Give the address back only now that the container is gone: releasing it while
- // Docker still holds the endpoint would hand the same IP to the next launch and
- // have Docker refuse it.
- vpcNetworkManager.releasePrivateIp(instance.getRegion(), instance.getSubnetId(),
- instance.getPrivateIpAddress());
- instance.setState(InstanceState.terminated());
- instance.setTerminatedAt(System.currentTimeMillis());
});
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
index db669c9012..f8cf62e777 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
@@ -33,6 +33,7 @@ import org.jboss.logging.Logger;
import com.fasterxml.jackson.core.type.TypeReference;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.RequestContext;
+import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.AwsRegions;
@@ -3123,6 +3124,14 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
public List<Map<String, String>> terminateInstances(String region, List<String> instanceIds) {
ensureDefaultResources(region);
List<Map<String, String>> result = new ArrayList<>();
+ // Every instance named here is going away, so none of them counts as a live dependant of
+ // a capture -- not the ones already processed, which linger as shutting-down while their
+ // containers are torn down asynchronously, and not the ones still waiting their turn.
+ Set<String> terminating = new LinkedHashSet<>(instanceIds);
+ // Captured HERE, on the request thread, because the post-teardown hook below runs on the
+ // teardown executor where no request context is active and every account-aware store
+ // would silently fall back to the default account.
+ String owner = callerAccountId();
for (String id : instanceIds) {
Instance inst = getRequiredInstance(region, id);
@@ -3134,7 +3143,12 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
inst.setState(InstanceState.terminated());
inst.setTerminatedAt(System.currentTimeMillis());
} else {
- containerManager.terminate(inst);
+ // Also attempt the reclaim once this container is really gone. The synchronous
+ // attempt below runs while teardown is still scheduled, so for a batch whose
+ // containers all tear down slowly every in-loop attempt can be refused by the
+ // daemon and, without this, nothing would try again.
+ containerManager.terminate(inst, () ->
+ RequestScopes.runAs(owner, () -> reclaimCapturesPinnedBy(region, inst, terminating)));
}
// Delete root volume if deleteOnTermination (matches real AWS behavior)
if (inst.getRootVolumeId() != null) {
@@ -3143,7 +3157,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
releaseStandaloneInterfacesOnTermination(region, inst);
instances.put(key(region, id), inst);
// The last instance depending on a deregistered AMI's capture has just gone away.
- reclaimCapturesPinnedBy(region, inst);
+ // Cheap and usually sufficient: by here the container is often already removed.
+ reclaimCapturesPinnedBy(region, inst, terminating);
Map<String, String> entry = new LinkedHashMap<>();
entry.put("instanceId", id);
entry.put("previousState", prev.getName());
@@ -5510,13 +5525,13 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
*
* <p>Callers must hold {@link #imageRegistryLock}.
*
- * @param excludedInstanceId an instance not to count as a live dependant, used by the
- * terminate path where the store still reads the instance as
- * running while its container is being torn down; null to count
- * every live instance
+ * @param excludedInstanceIds the instances not to count as live dependants, used by the
+ * terminate path where the store still reads every instance of the
+ * request as running or shutting-down while their containers are
+ * being torn down; null or empty to count every live instance
* @return true when the reference was released, so the caller knows to store the change
*/
- private boolean reclaimCapturedImage(Image image, String excludedInstanceId) {
+ private boolean reclaimCapturedImage(Image image, Set<String> excludedInstanceIds) {
String captured = image.getDockerImage();
if (captured == null || config.services().ec2().mock()) {
return false;
@@ -5530,7 +5545,8 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
return false;
}
boolean stillLaunchable = instances.scan(i -> true).stream()
- .filter(i -> i.getRegion() != null && !i.getInstanceId().equals(excludedInstanceId))
+ .filter(i -> i.getRegion() != null
+ && (excludedInstanceIds == null || !excludedInstanceIds.contains(i.getInstanceId())))
.filter(i -> i.getState() != null && !"terminated".equals(i.getState().getName()))
.anyMatch(i -> captured.equals(capturedImageFor(i.getRegion(), i.getImageId())));
if (stillLaunchable) {
@@ -5554,8 +5570,13 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
*
* <p>Only tombstoned AMIs are considered: while the AMI is still registered its capture is
* needed for the next launch.
+ *
+ * <p>{@code terminating} is the whole batch of the current request, not just this instance.
+ * A reclaim attempted for the first of them can be refused by the daemon while the other
+ * containers still exist, and {@link #reclaimCapturedImage} only clears the reference on
+ * success, so the attempt comes round again on each remaining instance of the batch.
*/
- private void reclaimCapturesPinnedBy(String region, Instance terminated) {
+ private void reclaimCapturesPinnedBy(String region, Instance terminated, Set<String> terminating) {
if (config.services().ec2().mock()) {
return;
}
@@ -5569,7 +5590,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {
.filter(img -> DEREGISTERED_STATE.equals(img.getState()))
.findFirst()
.orElse(null);
- if (holder != null && reclaimCapturedImage(holder, terminated.getInstanceId())) {
+ if (holder != null && reclaimCapturedImage(holder, terminating)) {
registeredImages.put(key(holder.getRegion(), holder.getImageId()), holder);
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2CapturedImageReclaimIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2CapturedImageReclaimIntegrationTest.java
index e700eed32b..ca43e90b8e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2CapturedImageReclaimIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2CapturedImageReclaimIntegrationTest.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.ec2;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.services.ec2.model.Image;
import io.github.hectorvent.floci.services.ec2.model.Instance;
import io.github.hectorvent.floci.services.ec2.model.InstanceState;
@@ -13,6 +14,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -89,15 +91,31 @@ class Ec2CapturedImageReclaimIntegrationTest {
when(containerManager.removeCommittedImage(anyString())).thenReturn(true);
}
+ /**
+ * Post-teardown callbacks handed to the container manager and not yet run. Production runs
+ * these on the teardown executor once the container is really gone; a test that wants to
+ * model teardown still being in flight leaves them here and runs them when it chooses.
+ */
+ private final List<Runnable> pendingTeardowns = new ArrayList<>();
+
@BeforeEach
void terminationMarksInstancesTerminated() {
// Outside mock mode Ec2Service delegates termination to the container manager, which is
// mocked here, so without this an instance stays "running" in the store forever and any
// assertion about terminated instances would be vacuous.
+ pendingTeardowns.clear();
doAnswer(invocation -> {
invocation.<Instance>getArgument(0).setState(InstanceState.terminated());
+ runTeardownCallback(invocation.getArgument(1));
return null;
- }).when(containerManager).terminate(any(Instance.class));
+ }).when(containerManager).terminate(any(Instance.class), any());
+ }
+
+ /** Runs a post-teardown callback if one was supplied, mirroring the production finally block. */
+ private void runTeardownCallback(Object callback) {
+ if (callback instanceof Runnable runnable) {
+ runnable.run();
+ }
}
private Instance launch(String imageId) {
@@ -261,6 +279,11 @@ class Ec2CapturedImageReclaimIntegrationTest {
// clearing the reference on a transient docker failure leaks the layer permanently. The
// second termination is the later attempt: it only reaches the daemon at all if the
// reference survived the first failure.
+ //
+ // Counts: one termination reaches the daemon twice, once in the synchronous loop and
+ // once from the post-teardown callback. The second termination adds only one, because
+ // its synchronous attempt succeeds and clears the reference, so the callback that
+ // follows finds nothing left to reclaim.
String tag = "floci-ami/ami-removal-refused:latest";
Image image = captureAmi("removal-refused", tag);
Instance dependent = launch(image.getImageId());
@@ -269,12 +292,12 @@ class Ec2CapturedImageReclaimIntegrationTest {
when(containerManager.removeCommittedImage(tag)).thenReturn(false);
service.terminateInstances(REGION, List.of(dependent.getInstanceId()));
- verify(containerManager, times(1)).removeCommittedImage(tag);
+ verify(containerManager, times(2)).removeCommittedImage(tag);
when(containerManager.removeCommittedImage(tag)).thenReturn(true);
service.terminateInstances(REGION, List.of(dependent.getInstanceId()));
- verify(containerManager, times(2)).removeCommittedImage(tag);
+ verify(containerManager, times(3)).removeCommittedImage(tag);
}
// ─── Termination reclaims what deregistration had to retain ───────────────
@@ -308,6 +331,143 @@ class Ec2CapturedImageReclaimIntegrationTest {
verify(containerManager, never()).removeCommittedImage(tag);
}
+ /**
+ * Reproduces what production termination actually does to the store: the container manager
+ * flips the instance to {@code shutting-down} synchronously and only reaches
+ * {@code terminated} once its asynchronous teardown finishes. The default stub in this class
+ * terminates immediately, which hides every ordering question about a batch.
+ */
+ private void terminationLeavesInstancesShuttingDown() {
+ doAnswer(invocation -> {
+ invocation.<Instance>getArgument(0).setState(InstanceState.shuttingDown());
+ if (invocation.getArgument(1) instanceof Runnable callback) {
+ // Teardown has not finished, so the callback is held rather than run. A test that
+ // wants to model teardown completing drains pendingTeardowns itself.
+ pendingTeardowns.add(callback);
+ }
+ return null;
+ }).when(containerManager).terminate(any(Instance.class), any());
+ }
+
+ @Test
+ void terminatingAWholeBatchOfDependentsReleasesTheCapture() {
+ // Every instance in one TerminateInstances request goes away together, so none of them
+ // can pin the capture. Counting the siblings as live leaves the layer on disk forever:
+ // the reclaim runs only from termination and deregistration, deregistration is rejected
+ // the second time, and nothing revisits the batch after its teardown completes.
+ String tag = "floci-ami/ami-batch:latest";
+ Image image = captureAmi("batch-terminated", tag);
+ terminationLeavesInstancesShuttingDown();
+ Instance first = launch(image.getImageId());
+ Instance second = launch(image.getImageId());
+ service.deregisterImage(REGION, image.getImageId(), false);
+ verify(containerManager, never()).removeCommittedImage(tag);
+
+ service.terminateInstances(REGION,
+ List.of(first.getInstanceId(), second.getInstanceId()));
+
+ verify(containerManager).removeCommittedImage(tag);
+ }
+
+ @Test
+ void aBatchReclaimTheDaemonRefusesEarlyIsRetriedLaterInTheSameBatch() {
+ // With the whole batch excluded the reclaim is attempted on the first instance, while the
+ // other containers may still exist and Docker may refuse to delete the layer. The
+ // reference is only cleared on success, so the attempt has to come round again as the
+ // rest of the batch is processed.
+ String tag = "floci-ami/ami-batch-retry:latest";
+ Image image = captureAmi("batch-retry", tag);
+ terminationLeavesInstancesShuttingDown();
+ Instance first = launch(image.getImageId());
+ Instance second = launch(image.getImageId());
+ service.deregisterImage(REGION, image.getImageId(), false);
+ when(containerManager.removeCommittedImage(tag)).thenReturn(false, true);
+
+ service.terminateInstances(REGION,
+ List.of(first.getInstanceId(), second.getInstanceId()));
+
+ verify(containerManager, times(2)).removeCommittedImage(tag);
+ }
+
+ @Test
+ void aBatchWhoseTeardownOutlastsTheLoopIsReclaimedOnceTeardownCompletes() {
+ // The in-loop attempts are not enough on their own. Container teardown is asynchronous,
+ // so a batch whose containers are all still present while the loop runs has every
+ // attempt refused by the daemon, and the loop then ends. Without an attempt after
+ // teardown the layer would stay on disk for the lifetime of the emulator.
+ String tag = "floci-ami/ami-batch-slow-teardown:latest";
+ Image image = captureAmi("batch-slow-teardown", tag);
+ terminationLeavesInstancesShuttingDown();
+ Instance first = launch(image.getImageId());
+ Instance second = launch(image.getImageId());
+ service.deregisterImage(REGION, image.getImageId(), false);
+ // The daemon refuses while any container of the batch still exists.
+ when(containerManager.removeCommittedImage(tag)).thenReturn(false);
+
+ service.terminateInstances(REGION,
+ List.of(first.getInstanceId(), second.getInstanceId()));
+
+ // Every synchronous attempt was refused, so the layer is still held.
+ verify(containerManager, times(2)).removeCommittedImage(tag);
+
+ // Teardown completes. The containers are gone, so the daemon now accepts the removal.
+ when(containerManager.removeCommittedImage(tag)).thenReturn(true);
+ pendingTeardowns.forEach(Runnable::run);
+
+ verify(containerManager, times(3)).removeCommittedImage(tag);
+ }
+
+ @Test
+ void aBatchInANonDefaultAccountStillReclaimsAfterTeardown() throws Exception {
+ // The hook runs on the teardown executor, which carries no request context, so every
+ // account-aware store it touches falls back to the default account. For a function of
+ // this shape that is silent: capturedImageFor finds no registered image under
+ // 000000000000, returns null, and the reclaim returns at its first guard with nothing
+ // logged -- which is indistinguishable from "this AMI was never captured".
+ //
+ // The other batch tests cannot catch it. They drain pendingTeardowns on the test thread,
+ // where the request context IS active, so the fallback never happens. This one drains on
+ // a bare thread, which is what the executor actually gives the hook.
+ String tag = "floci-ami/ami-other-account:latest";
+ String otherAccount = "210987654321";
+
+ Image image = RequestScopes.callAs(otherAccount, () -> {
+ when(containerManager.commitInstance(any(Instance.class), anyString())).thenReturn(tag);
+ Instance source = launch(BASE_AMI);
+ Image captured = service.createImage(REGION, source.getInstanceId(),
+ "other-account", "captured", true);
+ service.terminateInstances(REGION, List.of(source.getInstanceId()));
+ return captured;
+ });
+
+ RequestScopes.runAs(otherAccount, () -> {
+ terminationLeavesInstancesShuttingDown();
+ Instance first = launch(image.getImageId());
+ Instance second = launch(image.getImageId());
+ service.deregisterImage(REGION, image.getImageId(), false);
+ when(containerManager.removeCommittedImage(tag)).thenReturn(false);
+ service.terminateInstances(REGION,
+ List.of(first.getInstanceId(), second.getInstanceId()));
+ });
+
+ // The two synchronous attempts ran on the request thread, where the account IS active, so
+ // they found the AMI and were refused by the daemon rather than by a wrong-account lookup.
+ // That is exactly why this bug is invisible to the other batch tests.
+ verify(containerManager, times(2)).removeCommittedImage(tag);
+
+ // Teardown finishes on a thread with no request scope at all, as it does in production.
+ when(containerManager.removeCommittedImage(tag)).thenReturn(true);
+ List<Runnable> draining = List.copyOf(pendingTeardowns);
+ Thread bare = new Thread(() -> draining.forEach(Runnable::run), "teardown-no-request-scope");
+ bare.start();
+ bare.join(30_000);
+
+ // The third attempt is the hook's. Without the account captured on the request thread it
+ // never happens: the lookup resolves under the default account, finds nothing, and the
+ // reclaim returns at its first guard, leaving the layer on disk for good.
+ verify(containerManager, times(3)).removeCommittedImage(tag);
+ }
+
@Test
void terminatingAnInstanceOfAStillRegisteredAmiKeepsTheCapture() {
// The AMI can still be launched again; its capture is not garbage.
diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java
index d1dbb59895..724612d1d1 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java
@@ -58,6 +58,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BooleanSupplier;
@@ -1783,6 +1784,41 @@ class Ec2ContainerManagerTest {
awaitUntil(() -> "running".equals(instance.getState().getName()), Duration.ofSeconds(2));
}
+ @Test
+ void theTeardownHookRunsOnceTheContainerIsGone() throws Exception {
+ // The hook exists so callers can do work that is only valid after the container has
+ // released its Docker resources, so it must actually fire on the ordinary path.
+ LaunchHarness harness = launchHarness();
+ stubDockerPing(harness.dockerClient(), false);
+ Instance instance = instance("i-teardown-hook");
+ harness.manager().launch(instance, "ubuntu:24.04", null, "us-west-2");
+ AtomicBoolean ran = new AtomicBoolean(false);
+
+ harness.manager().terminate(instance, () -> ran.set(true));
+
+ awaitUntil(ran::get, Duration.ofSeconds(2));
+ }
+
+ @Test
+ void theTeardownHookIsSkippedWhenTheContainerCouldNotBeRemoved() throws Exception {
+ // Running it anyway would start work that depends on the container being gone while it
+ // demonstrably is not, and nothing retries behind it. Skipping and logging is the
+ // honest outcome.
+ LaunchHarness harness = launchHarness();
+ stubDockerPing(harness.dockerClient(), false);
+ Instance instance = instance("i-teardown-hook-refused");
+ harness.manager().launch(instance, "ubuntu:24.04", null, "us-west-2");
+ instance.setDockerContainerId("c-stuck");
+ when(harness.dockerClient().removeContainerCmd("c-stuck"))
+ .thenThrow(new RuntimeException("daemon refused"));
+ AtomicBoolean ran = new AtomicBoolean(false);
+
+ harness.manager().terminate(instance, () -> ran.set(true));
+
+ awaitUntil(() -> "terminated".equals(instance.getState().getName()), Duration.ofSeconds(2));
+ assertFalse(ran.get(), "the hook must not run when the container is still present");
+ }
+
@Test
void metadataOnlyInstanceStopsStartsAndTerminatesWithoutAContainer() throws Exception {
LaunchHarness harness = launchHarness();