Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lifecycle): tear down process-bound containers on shutdown (#1869)

Hector Venturaeae039fbd056
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java b/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java
new file mode 100644
index 0000000000..2d5e0211e7
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java
@@ -0,0 +1,19 @@
+package io.github.hectorvent.floci.core.common;
+
+/**
+ * Implemented by services that launch Docker containers whose lifetime is bound to the
+ * emulator process (Lambda warm pool, ECS tasks, EC2 instances, in-flight build/job
+ * containers). {@code EmulatorLifecycle.onStop} invokes every implementation during the
+ * ShutdownEvent phase — before {@code StorageFactory.shutdownAll()} — so teardown runs at
+ * a deterministic point and state changes made while stopping are still captured by the
+ * final flush. A {@code @PreDestroy} alone is not sufficient for that: bean destruction
+ * runs after the ShutdownEvent observers, when the storage flush schedulers are already
+ * stopped.
+ *
+ * <p>Implementations must be idempotent; they may also be invoked from {@code @PreDestroy}
+ * as a fallback.
+ */
+public interface ContainerTeardown {
+
+ void stopManagedContainers();
+}
diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
index 498835fc87..5c1cb5f51c 100644
--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.lifecycle;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.ContainerTeardown;
import io.github.hectorvent.floci.core.common.ServiceRegistry;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.lifecycle.inithook.InitializationHook;
@@ -83,6 +84,7 @@ public class EmulatorLifecycle {
private final FlociUiManager flociUiManager;
private final InitLifecycleState initLifecycleState;
private final SchemaCreationWorker schemaCreationWorker;
+ private final jakarta.enterprise.inject.Instance<ContainerTeardown> containerTeardowns;
@Inject
public EmulatorLifecycle(StorageFactory storageFactory, ServiceRegistry serviceRegistry,
@@ -108,7 +110,8 @@ public class EmulatorLifecycle {
EcrRegistryManager ecrRegistryManager,
FlociUiManager flociUiManager,
InitLifecycleState initLifecycleState,
- SchemaCreationWorker schemaCreationWorker) {
+ SchemaCreationWorker schemaCreationWorker,
+ jakarta.enterprise.inject.Instance<ContainerTeardown> containerTeardowns) {
this.storageFactory = storageFactory;
this.serviceRegistry = serviceRegistry;
this.config = config;
@@ -134,6 +137,7 @@ public class EmulatorLifecycle {
this.flociUiManager = flociUiManager;
this.initLifecycleState = initLifecycleState;
this.schemaCreationWorker = schemaCreationWorker;
+ this.containerTeardowns = containerTeardowns;
}
void onStart(@Observes StartupEvent ignored) {
@@ -264,6 +268,17 @@ public class EmulatorLifecycle {
rabbitMqManager.stopAll();
ecrRegistryManager.shutdown();
flociUiManager.shutdown();
+ // Centralized teardown for process-bound containers (Lambda warm pool, ECS tasks,
+ // EC2 instances, in-flight build/job containers). Runs before shutdownAll() so any
+ // state written while stopping is captured by the final flush.
+ for (ContainerTeardown teardown : containerTeardowns) {
+ try {
+ teardown.stopManagedContainers();
+ } catch (Exception e) {
+ LOG.warnv("Container teardown failed for {0}: {1}",
+ teardown.getClass().getSimpleName(), e.getMessage());
+ }
+ }
storageFactory.shutdownAll();
LOG.info("=== AWS Local Emulator Stopped ===");
diff --git a/src/main/java/io/github/hectorvent/floci/services/batch/BatchDockerRunner.java b/src/main/java/io/github/hectorvent/floci/services/batch/BatchDockerRunner.java
index d5b62cddc2..838b23b4dc 100644
--- a/src/main/java/io/github/hectorvent/floci/services/batch/BatchDockerRunner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/batch/BatchDockerRunner.java
@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.batch;
import com.github.dockerjava.api.command.InspectContainerResponse;
import com.github.dockerjava.api.exception.NotFoundException;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.ContainerTeardown;
import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
@@ -22,13 +23,19 @@ import java.io.Closeable;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
@ApplicationScoped
-public class BatchDockerRunner {
+public class BatchDockerRunner implements ContainerTeardown {
private static final Logger LOG = Logger.getLogger(BatchDockerRunner.class);
private static final String LOG_GROUP = "/aws/batch/job";
+ // Containers of jobs currently inside run(); drained on emulator shutdown so a
+ // SIGTERM mid-job does not orphan the container.
+ private final ConcurrentHashMap<String, String> inFlightContainers = new ConcurrentHashMap<>();
+
private final ContainerBuilder containerBuilder;
private final ContainerLifecycleManager lifecycleManager;
private final ContainerLogStreamer logStreamer;
@@ -76,27 +83,59 @@ public class BatchDockerRunner {
ContainerSpec spec = builder.build();
containerId = lifecycleManager.createAndStart(spec).containerId();
+ inFlightContainers.put(job.getJobId(), containerId);
logHandle = logStreamer.attach(containerId, LOG_GROUP, logStreamName, job.getRegion(),
"batch:" + job.getJobName() + ":" + job.getJobId());
Integer exitCode = waitForExit(containerId, timeout(job));
long stoppedAt = System.currentTimeMillis();
+ releaseAndStop(job.getJobId(), containerId, logHandle);
if (exitCode == null) {
- lifecycleManager.stopAndRemove(containerId, logHandle);
return new BatchRunResult(137, "Job timed out", logStreamName, startedAt, stoppedAt, true);
}
- lifecycleManager.stopAndRemove(containerId, logHandle);
return new BatchRunResult(exitCode, exitCode == 0 ? null : "Container exited with code " + exitCode,
logStreamName, startedAt, stoppedAt, false);
} catch (Exception e) {
LOG.warnv("Batch Docker job {0} failed: {1}", job.getJobId(), e.getMessage());
if (containerId != null) {
- lifecycleManager.stopAndRemove(containerId, logHandle);
+ releaseAndStop(job.getJobId(), containerId, logHandle);
}
return failed(startedAt, logStreamName, e.getMessage());
}
}
+ // Whoever wins the map removal owns the stop — stopManagedContainers() may have
+ // claimed the container first during shutdown, but the log stream is still ours.
+ private void releaseAndStop(String jobId, String containerId, Closeable logHandle) {
+ if (inFlightContainers.remove(jobId, containerId)) {
+ lifecycleManager.stopAndRemove(containerId, logHandle);
+ } else if (logHandle != null) {
+ try {
+ logHandle.close();
+ } catch (Exception e) {
+ LOG.debugv("Error closing log stream for job {0}: {1}", jobId, e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * Stops the containers of jobs still inside {@link #run} on emulator shutdown;
+ * without this a SIGTERM mid-job orphans the container.
+ */
+ @Override
+ public void stopManagedContainers() {
+ for (Map.Entry<String, String> entry : new ConcurrentHashMap<>(inFlightContainers).entrySet()) {
+ if (inFlightContainers.remove(entry.getKey(), entry.getValue())) {
+ try {
+ lifecycleManager.stopAndRemove(entry.getValue(), null);
+ } catch (Exception e) {
+ LOG.warnv("Failed to stop Batch container for job {0} on shutdown: {1}",
+ entry.getKey(), e.getMessage());
+ }
+ }
+ }
+ }
+
private BatchRunResult failed(long startedAt, String logStreamName, String reason) {
return new BatchRunResult(1, reason, logStreamName, startedAt, System.currentTimeMillis(), false);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
index e75072ffdc..1547f6ad1e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
@@ -5,6 +5,7 @@ import com.github.dockerjava.api.async.ResultCallback;
import com.github.dockerjava.api.model.Frame;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
+import io.github.hectorvent.floci.core.common.ContainerTeardown;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
@@ -54,7 +55,7 @@ import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
@ApplicationScoped
-public class CodeBuildRunner {
+public class CodeBuildRunner implements ContainerTeardown {
private static final Logger LOG = Logger.getLogger(CodeBuildRunner.class);
@@ -95,6 +96,25 @@ public class CodeBuildRunner {
this.regionResolver = regionResolver;
}
+ /**
+ * Stops the containers of all in-flight builds on emulator shutdown; without this
+ * they outlive the process as orphans. Build state is transient, so there is no
+ * persisted status to update.
+ */
+ @Override
+ public void stopManagedContainers() {
+ for (Map.Entry<String, String> entry : new LinkedHashMap<>(runningContainers).entrySet()) {
+ if (runningContainers.remove(entry.getKey(), entry.getValue())) {
+ try {
+ lifecycleManager.stopAndRemove(entry.getValue(), null);
+ } catch (Exception e) {
+ LOG.warnv("Failed to stop CodeBuild container for build {0} on shutdown: {1}",
+ entry.getKey(), e.getMessage());
+ }
+ }
+ }
+ }
+
public void startBuild(String region, Build build, Project project, String buildspecOverride) {
AtomicBoolean stopFlag = new AtomicBoolean(false);
stopFlags.put(build.getId(), stopFlag);
@@ -349,11 +369,10 @@ public class CodeBuildRunner {
}
} finally {
stopFlags.remove(buildId);
- if (containerId != null) {
- runningContainers.remove(buildId);
- if (logHandle != null) {
- try { logHandle.close(); } catch (Exception ignored) {}
- }
+ if (logHandle != null) {
+ try { logHandle.close(); } catch (Exception ignored) {}
+ }
+ if (containerId != null && runningContainers.remove(buildId, containerId)) {
lifecycleManager.stopAndRemove(containerId, null);
}
if (workspace != null) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java
index 7d06f71302..f8812cd0b1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java
@@ -239,6 +239,27 @@ public class Ec2ContainerManager {
});
}
+ /**
+ * Synchronous stop for emulator shutdown: tears down the port-forward sidecars and
+ * stops the container with a short timeout so N instances cannot exhaust the SIGTERM
+ * grace window. Unlike {@link #stop}, runs on the caller's thread (the async executor
+ * would be abandoned mid-flight during shutdown) and leaves state handling to the caller.
+ */
+ public void stopForShutdown(Instance instance) {
+ String containerId = instance.getDockerContainerId();
+ if (containerId == null) {
+ return;
+ }
+ portForwardManager.unpublishAll(instance);
+ try {
+ dockerClient.stopContainerCmd(containerId).withTimeout(5).exec();
+ } catch (NotFoundException e) {
+ // already gone
+ } catch (Exception e) {
+ LOG.warnv("Error stopping EC2 container {0} on shutdown: {1}", containerId, e.getMessage());
+ }
+ }
+
/**
* Gracefully stops a running container (30 second timeout then SIGKILL).
* Updates instance state through stopping → stopped.
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
index 99f8b28aa4..a80c0d0547 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
@@ -27,6 +27,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.AwsRegions;
+import io.github.hectorvent.floci.core.common.ContainerTeardown;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.ec2.model.Address;
@@ -77,7 +78,7 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
@ApplicationScoped
-public class Ec2Service {
+public class Ec2Service implements ContainerTeardown {
private static final Logger LOG = Logger.getLogger(Ec2Service.class);
private static final DateTimeFormatter ISO_FMT = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
@@ -902,6 +903,34 @@ public class Ec2Service {
return result;
}
+ /**
+ * Stops the Docker containers of running instances on emulator shutdown. Without this
+ * they outlive the process as orphans. Instances flip to {@code stopped} — the container
+ * really is stopped, and the id is kept so StartInstances can revive it after a restart.
+ * Runs during the ShutdownEvent phase, so the state change is captured by the final flush.
+ */
+ @Override
+ public void stopManagedContainers() {
+ if (config.services().ec2().mock()) {
+ return;
+ }
+ for (String storeKey : Set.copyOf(instances.keys())) {
+ Instance inst = instances.get(storeKey).orElse(null);
+ if (inst == null || inst.getDockerContainerId() == null
+ || inst.getState() == null || !"running".equals(inst.getState().getName())) {
+ continue;
+ }
+ try {
+ containerManager.stopForShutdown(inst);
+ inst.setState(InstanceState.stopped());
+ instances.put(storeKey, inst);
+ } catch (Exception e) {
+ LOG.warnv("Failed to stop EC2 instance container {0} on shutdown: {1}",
+ inst.getDockerContainerId(), e.getMessage());
+ }
+ }
+ }
+
public List<Map<String, String>> stopInstances(String region, List<String> instanceIds) {
ensureDefaultResources(region);
List<Map<String, String>> result = new ArrayList<>();
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index caea87b84e..c088d6e795 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.ecs;
import com.fasterxml.jackson.core.type.TypeReference;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.ContainerTeardown;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.StorageBackedMap;
import io.github.hectorvent.floci.core.storage.StorageFactory;
@@ -49,7 +50,7 @@ import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
@ApplicationScoped
-public class EcsService {
+public class EcsService implements ContainerTeardown {
private static final Logger LOG = Logger.getLogger(EcsService.class);
private static final String DEFAULT_CLUSTER = "default";
@@ -159,7 +160,40 @@ public class EcsService {
@PreDestroy
void shutdown() {
+ stopManagedContainers();
+ }
+
+ /**
+ * Stops the Docker containers of all still-running tasks on emulator shutdown.
+ * Task state is transient (memory-only), so without this the containers outlive
+ * the process as orphans. The reconciler is shut down first — and any in-flight
+ * tick awaited — so it cannot restart drained tasks between this teardown and
+ * the final storage flush. Handles are claimed atomically to avoid racing an
+ * explicit StopTask.
+ */
+ @Override
+ public void stopManagedContainers() {
reconciler.shutdownNow();
+ try {
+ reconciler.awaitTermination(2, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ for (String taskArn : new ArrayList<>(taskHandles.keySet())) {
+ EcsTaskHandle claimed = taskHandles.remove(taskArn);
+ if (claimed == null) {
+ continue;
+ }
+ try {
+ containerManager.stopTask(claimed);
+ } catch (Exception e) {
+ LOG.warnv("Failed to stop ECS task {0} on shutdown: {1}", taskArn, e.getMessage());
+ }
+ }
+ }
+
+ boolean isReconcilerShutdown() {
+ return reconciler.isShutdown();
}
// ── Clusters ─────────────────────────────────────────────────────────────
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
index 6e7ec85a96..e6b4361d9a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.lambda;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.ContainerTeardown;
import io.github.hectorvent.floci.services.lambda.launcher.ContainerHandle;
import io.github.hectorvent.floci.services.lambda.launcher.ContainerLauncher;
import io.github.hectorvent.floci.services.lambda.model.ContainerState;
@@ -31,7 +32,7 @@ import java.util.concurrent.TimeUnit;
* after the invocation completes.
*/
@ApplicationScoped
-public class WarmPool {
+public class WarmPool implements ContainerTeardown {
private static final Logger LOG = Logger.getLogger(WarmPool.class);
@@ -43,7 +44,6 @@ public class WarmPool {
private final ConcurrentHashMap<String, ArrayDeque<ContainerHandle>> pool = new ConcurrentHashMap<>();
private final ScheduledExecutorService evictionScheduler = Executors.newSingleThreadScheduledExecutor(
r -> { Thread t = new Thread(r, "warm-pool-evictor"); t.setDaemon(true); return t; });
- private Thread shutdownHook;
@Inject
public WarmPool(ContainerLauncher containerLauncher, EmulatorConfig config) {
@@ -77,19 +77,21 @@ public class WarmPool {
LOG.infov("Lambda containers running in ephemeral mode (destroyed after each invocation)");
}
- shutdownHook = new Thread(this::drainAll, "warm-pool-shutdown-hook");
- Runtime.getRuntime().addShutdownHook(shutdownHook);
+ }
+
+ /**
+ * Invoked from EmulatorLifecycle.onStop for a deterministic drain during the
+ * ShutdownEvent phase; the {@code @PreDestroy} below stays as an idempotent fallback.
+ * This replaces the previous raw JVM shutdown hook, which raced the Quarkus-managed
+ * shutdown sequence.
+ */
+ @Override
+ public void stopManagedContainers() {
+ drainAll();
}
@PreDestroy
void shutdown() {
- if (shutdownHook != null) {
- try {
- Runtime.getRuntime().removeShutdownHook(shutdownHook);
- } catch (IllegalStateException ignored) {
- // JVM already shutting down
- }
- }
evictionScheduler.shutdownNow();
drainAll();
}
diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
index ef86720356..d12429116b 100644
--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
@@ -78,6 +78,7 @@ class EmulatorLifecycleTest {
@Mock private InitLifecycleState initLifecycleState;
@Mock private EmulatorConfig.TlsConfig tlsConfig;
@Mock private io.github.hectorvent.floci.services.appsync.graphql.SchemaCreationWorker schemaCreationWorker;
+ @Mock private jakarta.enterprise.inject.Instance<io.github.hectorvent.floci.core.common.ContainerTeardown> containerTeardowns;
private EmulatorLifecycle emulatorLifecycle;
@@ -98,7 +99,9 @@ class EmulatorLifecycleTest {
rabbitMqManager, rdsService,
initializationHooksRunner, sqsPoller, kinesisPoller, dynamodbStreamsPoller,
pipesService, ec2MetadataServer, ecrRegistryManager, flociUiManager, initLifecycleState,
- schemaCreationWorker);
+ schemaCreationWorker, containerTeardowns);
+ Mockito.lenient().when(containerTeardowns.iterator())
+ .thenReturn(java.util.Collections.emptyIterator());
}
private void stubStorageConfig() {
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java
new file mode 100644
index 0000000000..b1d93235f4
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java
@@ -0,0 +1,90 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.ecs.container.EcsContainerManager;
+import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;
+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.LaunchType;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Verifies emulator-shutdown teardown stops the Docker containers of running tasks
+ * exactly once. Without it, task containers outlive the process as orphans (task
+ * state is transient, so nothing reclaims them on the next start).
+ */
+class EcsServiceTeardownTest {
+
+ private static final String REGION = "us-east-1";
+
+ @Test
+ void stopManagedContainersStopsEachRunningTaskOnce() {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().mock()).thenReturn(false); // docker mode
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+
+ EcsContainerManager containerManager = mock(EcsContainerManager.class);
+ EcsTaskHandle handle = mock(EcsTaskHandle.class);
+ when(containerManager.startTask(any(), any(), any(), anyString())).thenReturn(handle);
+
+ EcsService service = new EcsService(
+ new RegionResolver(REGION, "000000000000"),
+ containerManager,
+ config,
+ mock(EcsLoadBalancerRegistrar.class),
+ new SingleUseStorageFactory());
+ service.initializeStorage();
+
+ ContainerDefinition cd = new ContainerDefinition();
+ cd.setName("app");
+ cd.setImage("nginx:alpine");
+ service.registerTaskDefinition("teardown-fam", List.of(cd), null, null, null,
+ null, null, List.of(), REGION);
+ service.runTask(null, "teardown-fam", 1, LaunchType.FARGATE, null, null,
+ List.of(), null, REGION);
+
+ service.stopManagedContainers();
+ verify(containerManager, times(1)).stopTask(handle);
+
+ // The reconciler must be stopped before handles are drained, or a tick could
+ // restart the drained tasks between teardown and the final storage flush.
+ assertTrue(service.isReconcilerShutdown());
+
+ // Handles are claimed on the first pass; a second invocation must be a no-op.
+ service.stopManagedContainers();
+ verify(containerManager, times(1)).stopTask(handle);
+ }
+
+ private static final class SingleUseStorageFactory extends StorageFactory {
+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
+
+ private SingleUseStorageFactory() {
+ super(null, null);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> StorageBackend<String, V> create(String serviceName,
+ String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return (StorageBackend<String, V>) stores.computeIfAbsent(fileName, ignored -> new InMemoryStorage<>());
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
index dfa19e792e..e1d10754f0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
@@ -10,10 +10,7 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
-import java.lang.reflect.Field;
-
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.mockito.ArgumentMatchers.any;
@@ -40,29 +37,33 @@ class WarmPoolTest {
}
@Test
- void shutdownHookRegisteredAfterInit() throws Exception {
+ void stopManagedContainersDrainsPool() {
+ // Lifecycle-driven teardown replaces the old raw JVM shutdown hook: the pool
+ // drains when EmulatorLifecycle.onStop invokes the ContainerTeardown contract.
WarmPool pool = buildPool();
pool.init();
- Field hookField = WarmPool.class.getDeclaredField("shutdownHook");
- hookField.setAccessible(true);
- Thread hook = (Thread) hookField.get(pool);
+ LambdaFunction fn = mock(LambdaFunction.class);
+ when(fn.getFunctionName()).thenReturn("drain-fn");
+ ContainerHandle handle = new ContainerHandle("cid-drain", "drain-fn", null, ContainerState.WARM);
+ when(containerLauncher.launch(any())).thenReturn(handle);
+
+ pool.release(pool.acquire(fn));
+ pool.stopManagedContainers();
+ verify(containerLauncher).stop(handle);
- assertNotNull(hook);
+ // Idempotent: a second drain (e.g. the @PreDestroy fallback) is a no-op.
+ pool.stopManagedContainers();
+ verify(containerLauncher, times(1)).stop(handle);
pool.shutdown();
}
@Test
- void shutdownHookDrainsEmptyPool() throws Exception {
+ void stopManagedContainersOnEmptyPoolIsNoOp() {
WarmPool pool = buildPool();
pool.init();
- Field hookField = WarmPool.class.getDeclaredField("shutdownHook");
- hookField.setAccessible(true);
- Thread hook = (Thread) hookField.get(pool);
-
- // Running the hook on an empty pool must not throw
- hook.run();
+ pool.stopManagedContainers();
pool.shutdown();
}