Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(sagemaker): restore the worker pools in afterReset, not clear() (#3937)

Matheus André52941e90ef70
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java b/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java
index 4632be5567..07013c9c86 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java
@@ -16,8 +16,10 @@ package io.github.hectorvent.floci.core.common;
*
* <p>Implementations must be idempotent; they may also be invoked from {@code @PreDestroy}
* as a fallback. An implementation that shuts down its own executor here must also implement
- * {@link Resettable} and replace that executor in {@code clear()}: a reset runs this hook
- * too, and {@code clear()} runs after it on a reset but never on shutdown.
+ * {@link Resettable} and replace that executor in {@code afterReset()}: a reset runs this hook
+ * too, and {@code afterReset()} runs at the end of a reset, even one whose storage wipe or
+ * {@code clear()} failed, but never on shutdown. {@code clear()} is not enough, since the
+ * controller skips it once an earlier step threw and the pool would stay terminated for good.
*/
public interface ContainerTeardown {
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index d00166cd8d..cb0500580a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -77,7 +77,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
private final EcsEventPublisher eventPublisher;
private final boolean dockerMode;
private final String baseUrl;
- // Replaced by clear() after a state reset, whose container teardown shuts this scheduler down.
+ // Replaced by afterReset() after a state reset, whose container teardown shuts this scheduler down.
private volatile ScheduledExecutorService reconciler = newReconciler();
private final Object reconcilerLock = new Object();
@@ -207,7 +207,7 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
* state reset. Task state is transient (memory-only), so without this the containers
* outlive the process as orphans. The reconciler is shut down first, and any in-flight
* tick awaited, so it cannot restart drained tasks between this teardown and the final
- * storage flush. A reset brings it back in {@link #clear()}. Handles are claimed
+ * storage flush. A reset brings it back in {@link #afterReset()}. Handles are claimed
* atomically to avoid racing an explicit StopTask.
*/
@Override
@@ -232,13 +232,20 @@ public class EcsService implements ContainerTeardown, ResourceProvider, Resettab
}
}
+ @Override
+ public void clear() {
+ // Nothing to wipe: the stores hold the ECS state and afterReset() restarts the reconciler.
+ }
+
/**
- * Runs after a state reset has drained the tasks and wiped the store, never on shutdown.
- * The reset's teardown stopped the reconciler, so without a new one no service would be
- * reconciled again until the emulator restarted.
+ * Runs at the end of every state reset, never on shutdown. The reset's teardown stopped the
+ * reconciler, so without a new one no service would be reconciled again until the emulator
+ * restarted. This hook rather than {@code clear()} because the controller runs it even when
+ * the storage wipe or another service's {@code clear()} threw, and a failed reset must not
+ * leave ECS without a reconciler for good.
*/
@Override
- public void clear() {
+ public void afterReset() {
synchronized (reconcilerLock) {
if (reconciler.isShutdown()) {
ScheduledExecutorService replacement = newReconciler();
diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java
index 581768b1e4..647e65346b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java
@@ -54,7 +54,7 @@ public class SageMakerEndpointManager implements ContainerTeardown, Resettable {
private final EmulatorConfig config;
private final ContainerDetector containerDetector;
private final S3Service s3Service;
- // Replaced by clear() after a state reset, whose container teardown shuts this pool down.
+ // Replaced by afterReset() after a state reset, whose container teardown shuts this pool down.
private volatile ExecutorService executor = Executors.newCachedThreadPool();
private final ConcurrentHashMap<String, String> containers = new ConcurrentHashMap<>();
private final HttpClient httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(2)).build();
@@ -152,13 +152,20 @@ public class SageMakerEndpointManager implements ContainerTeardown, Resettable {
executor.shutdownNow();
}
+ @Override
+ public void clear() {
+ // Nothing to wipe: the store holds the endpoints and afterReset() restores the pool.
+ }
+
/**
- * Runs after a state reset has torn the containers down and wiped the store, never on
- * shutdown. The teardown shut the worker pool down, so without a new one every later
- * CreateEndpoint and UpdateEndpoint would be rejected until the emulator restarted.
+ * Runs at the end of every state reset, never on shutdown. The teardown shut the worker
+ * pool down, so without a new one every later CreateEndpoint and UpdateEndpoint would be
+ * rejected until the emulator restarted. This hook rather than {@code clear()} because the
+ * controller runs it even when the storage wipe or another service's {@code clear()} threw,
+ * and a failed reset must not leave the pool terminated for good.
*/
@Override
- public synchronized void clear() {
+ public synchronized void afterReset() {
if (executor.isShutdown()) {
executor = Executors.newCachedThreadPool();
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java
index 94e010075a..76c979dc3f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java
@@ -54,7 +54,7 @@ public class SageMakerTrainingRunner implements ContainerTeardown, Resettable {
private final ContainerDetector containerDetector;
private final S3Service s3Service;
private final ObjectMapper mapper;
- // Replaced by clear() after a state reset, whose container teardown shuts this pool down.
+ // Replaced by afterReset() after a state reset, whose container teardown shuts this pool down.
private volatile ExecutorService executor = Executors.newCachedThreadPool();
private final ConcurrentHashMap<String, String> containers = new ConcurrentHashMap<>();
// Names a stop that stop() has already committed to the store as "Stopped": run()'s exit-code
@@ -170,14 +170,20 @@ public class SageMakerTrainingRunner implements ContainerTeardown, Resettable {
executor.shutdownNow();
}
- /**
- * Runs after a state reset has torn the containers down and wiped the store, never on
- * shutdown. The teardown shut the worker pool down, so without a new one every later
- * CreateTrainingJob would be rejected until the emulator restarted.
- */
@Override
public synchronized void clear() {
stopRequested.clear();
+ }
+
+ /**
+ * Runs at the end of every state reset, never on shutdown. The teardown shut the worker
+ * pool down, so without a new one every later CreateTrainingJob would be rejected until
+ * the emulator restarted. This hook rather than {@code clear()} because the controller
+ * runs it even when the storage wipe or another service's {@code clear()} threw, and a
+ * failed reset must not leave the pool terminated for good.
+ */
+ @Override
+ public synchronized void afterReset() {
if (executor.isShutdown()) {
executor = Executors.newCachedThreadPool();
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java
index ef666a60e7..bd93eb1f42 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java
@@ -78,7 +78,7 @@ class EcsServiceTeardownTest {
}
@Test
- void clearRestartsTheReconcilerThatAResetTeardownStopped() {
+ void afterResetRestartsTheReconcilerThatAResetTeardownStopped() {
EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
when(config.services().ecs().mock()).thenReturn(false); // docker mode
when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
@@ -91,13 +91,15 @@ class EcsServiceTeardownTest {
null);
service.initializeStorage();
- // A reset runs the teardown and then clear(); shutdown runs only the teardown.
+ // A reset runs the teardown, then clear(), and afterReset() last, even when the wipe or
+ // a clear() threw; shutdown runs only the teardown. afterReset() alone must bring the
+ // reconciler back.
service.stopManagedContainers();
assertTrue(service.isReconcilerShutdown());
- service.clear();
+ service.afterReset();
assertFalse(service.isReconcilerShutdown());
- service.clear();
+ service.afterReset();
assertFalse(service.isReconcilerShutdown());
service.stopManagedContainers();
diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManagerTest.java
index 8c5c27d139..4813a1b649 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManagerTest.java
@@ -18,22 +18,24 @@ import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
/**
- * A state reset runs the container teardown, which shuts the worker pool down, and then
- * {@code clear()}. Shutdown runs only the teardown.
+ * A state reset runs the container teardown, which shuts the worker pool down, then
+ * {@code clear()}, and {@code afterReset()} last, even when the wipe or a {@code clear()}
+ * failed. Shutdown runs only the teardown.
*/
class SageMakerEndpointManagerTest {
@Test
- void teardownStopsThePoolAndClearRestoresIt() {
+ void teardownStopsThePoolAndAfterResetRestoresIt() {
SageMakerEndpointManager manager = manager();
assertTrue(manager.acceptsWork());
manager.stopManagedContainers();
assertFalse(manager.acceptsWork());
- manager.clear();
+ // A reset whose wipe threw skips clear(); afterReset() alone must bring the pool back.
+ manager.afterReset();
assertTrue(manager.acceptsWork());
- manager.clear();
+ manager.afterReset();
assertTrue(manager.acceptsWork());
manager.stopManagedContainers();
@@ -44,6 +46,7 @@ class SageMakerEndpointManagerTest {
SageMakerEndpointManager manager = manager();
manager.stopManagedContainers();
manager.clear();
+ manager.afterReset();
SageMakerService service = mock(SageMakerService.class);
CountDownLatch published = new CountDownLatch(1);
diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunnerTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunnerTest.java
index 5b60a0bef1..16da79984d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunnerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunnerTest.java
@@ -21,22 +21,24 @@ import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
/**
- * A state reset runs the container teardown, which shuts the worker pool down, and then
- * {@code clear()}. Shutdown runs only the teardown.
+ * A state reset runs the container teardown, which shuts the worker pool down, then
+ * {@code clear()}, and {@code afterReset()} last, even when the wipe or a {@code clear()}
+ * failed. Shutdown runs only the teardown.
*/
class SageMakerTrainingRunnerTest {
@Test
- void teardownStopsThePoolAndClearRestoresIt() {
+ void teardownStopsThePoolAndAfterResetRestoresIt() {
SageMakerTrainingRunner runner = runner();
assertTrue(runner.acceptsWork());
runner.stopManagedContainers();
assertFalse(runner.acceptsWork());
- runner.clear();
+ // A reset whose wipe threw skips clear(); afterReset() alone must bring the pool back.
+ runner.afterReset();
assertTrue(runner.acceptsWork());
- runner.clear();
+ runner.afterReset();
assertTrue(runner.acceptsWork());
runner.stopManagedContainers();
@@ -47,6 +49,7 @@ class SageMakerTrainingRunnerTest {
SageMakerTrainingRunner runner = runner();
runner.stopManagedContainers();
runner.clear();
+ runner.afterReset();
SageMakerService service = mock(SageMakerService.class);
CountDownLatch reported = new CountDownLatch(1);