Light, fluffy, and always free - The AWS Local Emulator alternative
fix(sagemaker): restore the worker pools in afterReset, not clear() (#3937)
Matheus André52941e90ef70
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java b/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.javaindex 4632be5567..07013c9c86 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java@@ -16,8 +16,10 @@ package io.github.hectorvent.floci.core.common;** <p>Implementations must be idempotent; they may also be invoked from {@code @PreDestroy}* as a fallback. An implementation that shuts down its own executor here must also implement- * {@link Resettable} and replace that executor in {@code clear()}: a reset runs this hook- * too, and {@code clear()} runs after it on a reset but never on shutdown.+ * {@link Resettable} and replace that executor in {@code afterReset()}: a reset runs this hook+ * too, and {@code afterReset()} runs at the end of a reset, even one whose storage wipe or+ * {@code clear()} failed, but never on shutdown. {@code clear()} is not enough, since the+ * controller skips it once an earlier step threw and the pool would stay terminated for good.*/public interface ContainerTeardown {diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.javaindex d00166cd8d..cb0500580a 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java@@ -77,7 +77,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettabprivate final EcsEventPublisher eventPublisher;private final boolean dockerMode;private final String baseUrl;- // Replaced by clear() after a state reset, whose container teardown shuts this scheduler down.+ // Replaced by afterReset() after a state reset, whose container teardown shuts this scheduler down.private volatile ScheduledExecutorService reconciler = newReconciler();private final Object reconcilerLock = new Object();@@ -207,7 +207,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab* state reset. Task state is transient (memory-only), so without this the containers* outlive the process as orphans. The reconciler is shut down first, and any in-flight* tick awaited, so it cannot restart drained tasks between this teardown and the final- * storage flush. A reset brings it back in {@link #clear()}. Handles are claimed+ * storage flush. A reset brings it back in {@link #afterReset()}. Handles are claimed* atomically to avoid racing an explicit StopTask.*/@Override@@ -232,13 +232,20 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab}}+ @Override+ public void clear() {+ // Nothing to wipe: the stores hold the ECS state and afterReset() restarts the reconciler.+ }+/**- * Runs after a state reset has drained the tasks and wiped the store, never on shutdown.- * The reset's teardown stopped the reconciler, so without a new one no service would be- * reconciled again until the emulator restarted.+ * Runs at the end of every state reset, never on shutdown. The reset's teardown stopped the+ * reconciler, so without a new one no service would be reconciled again until the emulator+ * restarted. This hook rather than {@code clear()} because the controller runs it even when+ * the storage wipe or another service's {@code clear()} threw, and a failed reset must not+ * leave ECS without a reconciler for good.*/@Override- public void clear() {+ public void afterReset() {synchronized (reconcilerLock) {if (reconciler.isShutdown()) {ScheduledExecutorService replacement = newReconciler();diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.javaindex 581768b1e4..647e65346b 100644--- a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java@@ -54,7 +54,7 @@ public class SageMakerEndpointManager implements ContainerTeardown, Resettable {private final EmulatorConfig config;private final ContainerDetector containerDetector;private final S3Service s3Service;- // Replaced by clear() after a state reset, whose container teardown shuts this pool down.+ // Replaced by afterReset() after a state reset, whose container teardown shuts this pool down.private volatile ExecutorService executor = Executors.newCachedThreadPool();private final ConcurrentHashMap<String, String> containers = new ConcurrentHashMap<>();private final HttpClient httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(2)).build();@@ -152,13 +152,20 @@ public class SageMakerEndpointManager implements ContainerTeardown, Resettable {executor.shutdownNow();}+ @Override+ public void clear() {+ // Nothing to wipe: the store holds the endpoints and afterReset() restores the pool.+ }+/**- * Runs after a state reset has torn the containers down and wiped the store, never on- * shutdown. The teardown shut the worker pool down, so without a new one every later- * CreateEndpoint and UpdateEndpoint would be rejected until the emulator restarted.+ * Runs at the end of every state reset, never on shutdown. The teardown shut the worker+ * pool down, so without a new one every later CreateEndpoint and UpdateEndpoint would be+ * rejected until the emulator restarted. This hook rather than {@code clear()} because the+ * controller runs it even when the storage wipe or another service's {@code clear()} threw,+ * and a failed reset must not leave the pool terminated for good.*/@Override- public synchronized void clear() {+ public synchronized void afterReset() {if (executor.isShutdown()) {executor = Executors.newCachedThreadPool();}diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.javaindex 94e010075a..76c979dc3f 100644--- a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java@@ -54,7 +54,7 @@ public class SageMakerTrainingRunner implements ContainerTeardown, Resettable {private final ContainerDetector containerDetector;private final S3Service s3Service;private final ObjectMapper mapper;- // Replaced by clear() after a state reset, whose container teardown shuts this pool down.+ // Replaced by afterReset() after a state reset, whose container teardown shuts this pool down.private volatile ExecutorService executor = Executors.newCachedThreadPool();private final ConcurrentHashMap<String, String> containers = new ConcurrentHashMap<>();// Names a stop that stop() has already committed to the store as "Stopped": run()'s exit-code@@ -170,14 +170,20 @@ public class SageMakerTrainingRunner implements ContainerTeardown, Resettable {executor.shutdownNow();}- /**- * Runs after a state reset has torn the containers down and wiped the store, never on- * shutdown. The teardown shut the worker pool down, so without a new one every later- * CreateTrainingJob would be rejected until the emulator restarted.- */@Overridepublic synchronized void clear() {stopRequested.clear();+ }++ /**+ * Runs at the end of every state reset, never on shutdown. The teardown shut the worker+ * pool down, so without a new one every later CreateTrainingJob would be rejected until+ * the emulator restarted. This hook rather than {@code clear()} because the controller+ * runs it even when the storage wipe or another service's {@code clear()} threw, and a+ * failed reset must not leave the pool terminated for good.+ */+ @Override+ public synchronized void afterReset() {if (executor.isShutdown()) {executor = Executors.newCachedThreadPool();}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.javaindex ef666a60e7..bd93eb1f42 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java@@ -78,7 +78,7 @@ class EcsServiceTeardownTest {}@Test- void clearRestartsTheReconcilerThatAResetTeardownStopped() {+ void afterResetRestartsTheReconcilerThatAResetTeardownStopped() {EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);when(config.services().ecs().mock()).thenReturn(false); // docker modewhen(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");@@ -91,13 +91,15 @@ class EcsServiceTeardownTest {null);service.initializeStorage();- // A reset runs the teardown and then clear(); shutdown runs only the teardown.+ // A reset runs the teardown, then clear(), and afterReset() last, even when the wipe or+ // a clear() threw; shutdown runs only the teardown. afterReset() alone must bring the+ // reconciler back.service.stopManagedContainers();assertTrue(service.isReconcilerShutdown());- service.clear();+ service.afterReset();assertFalse(service.isReconcilerShutdown());- service.clear();+ service.afterReset();assertFalse(service.isReconcilerShutdown());service.stopManagedContainers();diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManagerTest.javaindex 8c5c27d139..4813a1b649 100644--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManagerTest.java@@ -18,22 +18,24 @@ import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.mock;/**- * A state reset runs the container teardown, which shuts the worker pool down, and then- * {@code clear()}. Shutdown runs only the teardown.+ * A state reset runs the container teardown, which shuts the worker pool down, then+ * {@code clear()}, and {@code afterReset()} last, even when the wipe or a {@code clear()}+ * failed. Shutdown runs only the teardown.*/class SageMakerEndpointManagerTest {@Test- void teardownStopsThePoolAndClearRestoresIt() {+ void teardownStopsThePoolAndAfterResetRestoresIt() {SageMakerEndpointManager manager = manager();assertTrue(manager.acceptsWork());manager.stopManagedContainers();assertFalse(manager.acceptsWork());- manager.clear();+ // A reset whose wipe threw skips clear(); afterReset() alone must bring the pool back.+ manager.afterReset();assertTrue(manager.acceptsWork());- manager.clear();+ manager.afterReset();assertTrue(manager.acceptsWork());manager.stopManagedContainers();@@ -44,6 +46,7 @@ class SageMakerEndpointManagerTest {SageMakerEndpointManager manager = manager();manager.stopManagedContainers();manager.clear();+ manager.afterReset();SageMakerService service = mock(SageMakerService.class);CountDownLatch published = new CountDownLatch(1);diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunnerTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunnerTest.javaindex 5b60a0bef1..16da79984d 100644--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunnerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunnerTest.java@@ -21,22 +21,24 @@ import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.mock;/**- * A state reset runs the container teardown, which shuts the worker pool down, and then- * {@code clear()}. Shutdown runs only the teardown.+ * A state reset runs the container teardown, which shuts the worker pool down, then+ * {@code clear()}, and {@code afterReset()} last, even when the wipe or a {@code clear()}+ * failed. Shutdown runs only the teardown.*/class SageMakerTrainingRunnerTest {@Test- void teardownStopsThePoolAndClearRestoresIt() {+ void teardownStopsThePoolAndAfterResetRestoresIt() {SageMakerTrainingRunner runner = runner();assertTrue(runner.acceptsWork());runner.stopManagedContainers();assertFalse(runner.acceptsWork());- runner.clear();+ // A reset whose wipe threw skips clear(); afterReset() alone must bring the pool back.+ runner.afterReset();assertTrue(runner.acceptsWork());- runner.clear();+ runner.afterReset();assertTrue(runner.acceptsWork());runner.stopManagedContainers();@@ -47,6 +49,7 @@ class SageMakerTrainingRunnerTest {SageMakerTrainingRunner runner = runner();runner.stopManagedContainers();runner.clear();+ runner.afterReset();SageMakerService service = mock(SageMakerService.class);CountDownLatch reported = new CountDownLatch(1);