Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lifecycle): keep SageMaker and ECS background work running after a state reset (#3797)
Hector Ventura08656347c707
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java b/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.javaindex b0964d15b5..4632be5567 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java@@ -15,7 +15,9 @@ package io.github.hectorvent.floci.core.common;* storage flush schedulers are already stopped.** <p>Implementations must be idempotent; they may also be invoked from {@code @PreDestroy}- * as a fallback.+ * as a fallback. An implementation that shuts down its own executor here must also implement+ * {@link Resettable} and replace that executor in {@code clear()}: a reset runs this hook+ * too, and {@code clear()} runs after it on a reset but never on shutdown.*/public interface ContainerTeardown {diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.javaindex 0e710c855d..ac37ef3712 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.ContainerTeardown;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageBackedMap;@@ -63,7 +64,7 @@ import io.github.hectorvent.floci.core.resource.SupportedResourceType;import io.github.hectorvent.floci.core.common.AwsArnUtils;@ApplicationScoped-public class EcsService implements ContainerTeardown, ResourceProvider {+public class EcsService implements ContainerTeardown, ResourceProvider, Resettable {private static final Logger LOG = Logger.getLogger(EcsService.class);private static final String DEFAULT_CLUSTER = "default";@@ -75,8 +76,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider {private final EcsEventPublisher eventPublisher;private final boolean dockerMode;private final String baseUrl;- private final ScheduledExecutorService reconciler = Executors.newSingleThreadScheduledExecutor(- r -> { Thread t = new Thread(r, "ecs-reconciler"); t.setDaemon(true); return t; });+ // Replaced by clear() after a state reset, whose container teardown shuts this scheduler down.+ private volatile ScheduledExecutorService reconciler = newReconciler();+ private final Object reconcilerLock = new Object();// region::clusterName → EcsClusterprivate Map<String, EcsCluster> clusters = new ConcurrentHashMap<>();@@ -132,7 +134,16 @@ public class EcsService implements ContainerTeardown, ResourceProvider {@PostConstructvoid init() {initializeStorage();- reconciler.scheduleAtFixedRate(this::reconcile, 5, 5, TimeUnit.SECONDS);+ scheduleReconciliation(reconciler);+ }++ private void scheduleReconciliation(ScheduledExecutorService scheduler) {+ scheduler.scheduleAtFixedRate(this::reconcile, 5, 5, TimeUnit.SECONDS);+ }++ private static ScheduledExecutorService newReconciler() {+ return Executors.newSingleThreadScheduledExecutor(+ r -> { Thread t = new Thread(r, "ecs-reconciler"); t.setDaemon(true); return t; });}void initializeStorage() {@@ -191,18 +202,19 @@ public class EcsService implements ContainerTeardown, ResourceProvider {}/**- * Stops the Docker containers of all still-running tasks on emulator shutdown.- * Task state is transient (memory-only), so without this the containers outlive- * the process as orphans. The reconciler is shut down first — and any in-flight- * tick awaited — so it cannot restart drained tasks between this teardown and- * the final storage flush. Handles are claimed atomically to avoid racing an- * explicit StopTask.+ * Stops the Docker containers of all still-running tasks on emulator shutdown and on a+ * state reset. Task state is transient (memory-only), so without this the containers+ * outlive the process as orphans. The reconciler is shut down first, and any in-flight+ * tick awaited, so it cannot restart drained tasks between this teardown and the final+ * storage flush. A reset brings it back in {@link #clear()}. Handles are claimed+ * atomically to avoid racing an explicit StopTask.*/@Overridepublic void stopManagedContainers() {- reconciler.shutdownNow();+ ScheduledExecutorService current = reconciler;+ current.shutdownNow();try {- reconciler.awaitTermination(2, TimeUnit.SECONDS);+ current.awaitTermination(2, TimeUnit.SECONDS);} catch (InterruptedException e) {Thread.currentThread().interrupt();}@@ -219,6 +231,22 @@ public class EcsService implements ContainerTeardown, ResourceProvider {}}+ /**+ * Runs after a state reset has drained the tasks and wiped the store, never on shutdown.+ * The reset's teardown stopped the reconciler, so without a new one no service would be+ * reconciled again until the emulator restarted.+ */+ @Override+ public void clear() {+ synchronized (reconcilerLock) {+ if (reconciler.isShutdown()) {+ ScheduledExecutorService replacement = newReconciler();+ scheduleReconciliation(replacement);+ reconciler = replacement;+ }+ }+ }+boolean isReconcilerShutdown() {return reconciler.isShutdown();}diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.javaindex ea4c9e2665..581768b1e4 100644--- a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.sagemaker;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.ContainerTeardown;+import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerDetector;@@ -41,7 +42,7 @@ import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;@ApplicationScoped-public class SageMakerEndpointManager implements ContainerTeardown {+public class SageMakerEndpointManager implements ContainerTeardown, Resettable {private static final Logger LOG = Logger.getLogger(SageMakerEndpointManager.class);private static final int PORT = 8080;private static final int DEFAULT_FILE_MODE = 0644;@@ -53,7 +54,8 @@ public class SageMakerEndpointManager implements ContainerTeardown {private final EmulatorConfig config;private final ContainerDetector containerDetector;private final S3Service s3Service;- private final ExecutorService executor = Executors.newCachedThreadPool();+ // Replaced by clear() after a state reset, whose container teardown shuts this pool down.+ private volatile ExecutorService executor = Executors.newCachedThreadPool();private final ConcurrentHashMap<String, String> containers = new ConcurrentHashMap<>();private final HttpClient httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(2)).build();@@ -144,12 +146,28 @@ public class SageMakerEndpointManager implements ContainerTeardown {}@Override- public void stopManagedContainers() {+ public synchronized void stopManagedContainers() {containers.forEach((name, id) -> lifecycleManager.stopAndRemove(id, null));containers.clear();executor.shutdownNow();}+ /**+ * Runs after a state reset has torn the containers down and wiped the store, never on+ * shutdown. The teardown shut the worker pool down, so without a new one every later+ * CreateEndpoint and UpdateEndpoint would be rejected until the emulator restarted.+ */+ @Override+ public synchronized void clear() {+ if (executor.isShutdown()) {+ executor = Executors.newCachedThreadPool();+ }+ }++ boolean acceptsWork() {+ return !executor.isShutdown();+ }+private List<String> environment(String region, Map<String, String> modelEnv) {List<String> env = new ArrayList<>();env.add("SAGEMAKER_BIND_TO_PORT=8080");diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerService.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerService.javaindex 8962b25c64..3aa8c87136 100644--- a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerService.java+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerService.java@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.Modimport io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;+import org.jboss.logging.Logger;import java.time.Clock;import java.util.ArrayList;@@ -22,12 +23,14 @@ import java.util.Comparator;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;+import java.util.Objects;import java.util.Optional;import java.util.function.BiConsumer;import java.util.function.Function;@ApplicationScopedpublic class SageMakerService {+ private static final Logger LOG = Logger.getLogger(SageMakerService.class);static final int DEFAULT_LIMIT = 100;static final int MAX_RESULTS_CEILING = 100;private static final ObjectMapper STATIC_MAPPER = new ObjectMapper();@@ -468,8 +471,23 @@ public class SageMakerService {return true;}- public synchronized void updateTrainingJob(TrainingJobResource job) {- trainingJobStore.put(regionKey(job.region, job.trainingJobName), job);+ /**+ * Publishes a training run's progress or outcome only while the job the run started from is+ * still stored. A state reset interrupts in-flight runs and wipes the store, and an+ * interrupted run's failure write must not bring the wiped job back, whether it lands before+ * or after the wipe, nor overwrite a same-named job created after the reset.+ */+ public synchronized boolean updateTrainingJob(TrainingJobResource job) {+ String key = regionKey(job.region, job.trainingJobName);+ TrainingJobResource current = trainingJobStore.get(key).orElse(null);+ if (current == null || !Objects.equals(current.trainingJobArn, job.trainingJobArn)+ || current.creationTime != job.creationTime) {+ LOG.debugv("SageMaker training job {0} is no longer the stored job; discarding its run update",+ job.trainingJobName);+ return false;+ }+ trainingJobStore.put(key, job);+ return true;}private Map<String, String> resourceTags(String arn) {diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.javaindex de6a063964..94e010075a 100644--- a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java@@ -7,6 +7,7 @@ import com.github.dockerjava.core.command.LogContainerResultCallback;import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.ContainerTeardown;+import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerDetector;@@ -42,7 +43,7 @@ import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;@ApplicationScoped-public class SageMakerTrainingRunner implements ContainerTeardown {+public class SageMakerTrainingRunner implements ContainerTeardown, Resettable {private static final Logger LOG = Logger.getLogger(SageMakerTrainingRunner.class);private static final String LOG_GROUP = "/aws/sagemaker/TrainingJobs";@@ -53,7 +54,8 @@ public class SageMakerTrainingRunner implements ContainerTeardown {private final ContainerDetector containerDetector;private final S3Service s3Service;private final ObjectMapper mapper;- private final ExecutorService executor = Executors.newCachedThreadPool();+ // Replaced by clear() after a state reset, whose container teardown shuts this pool down.+ private volatile ExecutorService executor = Executors.newCachedThreadPool();private final ConcurrentHashMap<String, String> containers = new ConcurrentHashMap<>();// Names a stop that stop() has already committed to the store as "Stopped": run()'s exit-code// handling checks this before persisting, since a stop races the removal of the very container@@ -146,6 +148,10 @@ public class SageMakerTrainingRunner implements ContainerTeardown {job.failureReason = e.getMessage();job.trainingEndTime = System.currentTimeMillis();service.updateTrainingJob(job);+ if (e instanceof InterruptedException) {+ // Interrupted by a teardown's shutdownNow(): keep the flag for the pool thread.+ Thread.currentThread().interrupt();+ }}}@@ -158,12 +164,29 @@ public class SageMakerTrainingRunner implements ContainerTeardown {}@Override- public void stopManagedContainers() {+ public synchronized void stopManagedContainers() {containers.forEach((name, id) -> lifecycleManager.stopAndRemove(id, null));containers.clear();executor.shutdownNow();}+ /**+ * Runs after a state reset has torn the containers down and wiped the store, never on+ * shutdown. The teardown shut the worker pool down, so without a new one every later+ * CreateTrainingJob would be rejected until the emulator restarted.+ */+ @Override+ public synchronized void clear() {+ stopRequested.clear();+ if (executor.isShutdown()) {+ executor = Executors.newCachedThreadPool();+ }+ }++ boolean acceptsWork() {+ return !executor.isShutdown();+ }+private List<String> environment(TrainingJobResource job) {List<String> env = new ArrayList<>();env.add("TRAINING_JOB_NAME=" + job.trainingJobName);diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.javaindex 454a64e4d0..ef666a60e7 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java@@ -19,6 +19,7 @@ import java.util.List;import java.util.Map;import java.util.concurrent.atomic.AtomicInteger;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyString;@@ -76,6 +77,33 @@ class EcsServiceTeardownTest {verify(containerManager, times(1)).stopTask(handle);}+ @Test+ void clearRestartsTheReconcilerThatAResetTeardownStopped() {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().ecs().mock()).thenReturn(false); // docker mode+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ EcsService service = new EcsService(+ new RegionResolver(REGION, "000000000000"),+ mock(EcsContainerManager.class),+ config,+ mock(EcsLoadBalancerRegistrar.class),+ new SingleUseStorageFactory(),+ null);+ service.initializeStorage();++ // A reset runs the teardown and then clear(); shutdown runs only the teardown.+ service.stopManagedContainers();+ assertTrue(service.isReconcilerShutdown());++ service.clear();+ assertFalse(service.isReconcilerShutdown());+ service.clear();+ assertFalse(service.isReconcilerShutdown());++ service.stopManagedContainers();+ assertTrue(service.isReconcilerShutdown());+ }+@Testvoid stoppedTaskRetriesTeardownUntilItsRemainingLogStreamIsReleased() {EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManagerTest.javanew file mode 100644index 0000000000..8c5c27d139--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManagerTest.java@@ -0,0 +1,71 @@+package io.github.hectorvent.floci.services.sagemaker;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;+import org.junit.jupiter.api.Test;++import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.mock;++/**+ * A state reset runs the container teardown, which shuts the worker pool down, and then+ * {@code clear()}. Shutdown runs only the teardown.+ */+class SageMakerEndpointManagerTest {++ @Test+ void teardownStopsThePoolAndClearRestoresIt() {+ SageMakerEndpointManager manager = manager();+ assertTrue(manager.acceptsWork());++ manager.stopManagedContainers();+ assertFalse(manager.acceptsWork());++ manager.clear();+ assertTrue(manager.acceptsWork());+ manager.clear();+ assertTrue(manager.acceptsWork());++ manager.stopManagedContainers();+ }++ @Test+ void endpointStartsSubmittedAfterAResetStillRun() throws InterruptedException {+ SageMakerEndpointManager manager = manager();+ manager.stopManagedContainers();+ manager.clear();++ SageMakerService service = mock(SageMakerService.class);+ CountDownLatch published = new CountDownLatch(1);+ EndpointResource endpoint = new EndpointResource();+ endpoint.endpointName = "after-reset";+ endpoint.region = "us-east-1";+ // A latch rather than verify(timeout): finalizeEndpointStart is synchronized, and a timed+ // verify holds the mock's monitor while it polls, blocking the very call it waits for.+ doAnswer(invocation -> {+ published.countDown();+ return true;+ }).when(service).finalizeEndpointStart(endpoint);+ manager.startEndpointAsync(endpoint, service);++ // The mocked service has no model for the endpoint, so the start fails at once, and every+ // start that executes publishes its outcome: the publish proves the new pool ran the work.+ assertTrue(published.await(5, TimeUnit.SECONDS));+ manager.stopManagedContainers();+ }++ private static SageMakerEndpointManager manager() {+ return new SageMakerEndpointManager(mock(ContainerBuilder.class), mock(ContainerLifecycleManager.class),+ mock(EmulatorConfig.class, RETURNS_DEEP_STUBS), mock(ContainerDetector.class), mock(S3Service.class));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerResetIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerResetIntegrationTest.javanew file mode 100644index 0000000000..eea9aa351b--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerResetIntegrationTest.java@@ -0,0 +1,70 @@+package io.github.hectorvent.floci.services.sagemaker;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.Response;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.notNullValue;++/**+ * A state reset runs every container teardown, and the SageMaker training runner and endpoint+ * manager shut their worker pools down there. Without restoring them afterwards, every later+ * CreateTrainingJob, CreateEndpoint and UpdateEndpoint failed with a 500 until the emulator+ * restarted. The requests submit their work before touching Docker, so no daemon is needed.+ */+@QuarkusTest+class SageMakerResetIntegrationTest {+ private static final String AUTH =+ "AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/sagemaker/aws4_request";+ private static final String UNPULLABLE_IMAGE = "floci.invalid/none:latest";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void trainingJobsAndEndpointsCanStillBeCreatedAfterAStateReset() {+ given().when().post("/_floci/state/reset").then().statusCode(200);+ try {+ String suffix = Long.toString(System.nanoTime(), 36);+ post("SageMaker.CreateTrainingJob", """+ {"TrainingJobName":"sm-reset-%s",+ "AlgorithmSpecification":{"TrainingImage":"%s","TrainingInputMode":"File"},+ "OutputDataConfig":{"S3OutputPath":"s3://sm-reset-%s/output"},+ "ResourceConfig":{"InstanceType":"ml.m5.large","InstanceCount":1,"VolumeSizeInGB":1},+ "StoppingCondition":{"MaxRuntimeInSeconds":60}}+ """.formatted(suffix, UNPULLABLE_IMAGE, suffix))+ .then().statusCode(200).body("TrainingJobArn", notNullValue());++ String model = "sm-reset-model-" + suffix;+ post("SageMaker.CreateModel", """+ {"ModelName":"%s","PrimaryContainer":{"Image":"%s"}}+ """.formatted(model, UNPULLABLE_IMAGE))+ .then().statusCode(200);+ String config = "sm-reset-cfg-" + suffix;+ post("SageMaker.CreateEndpointConfig", """+ {"EndpointConfigName":"%s","ProductionVariants":[{"VariantName":"AllTraffic","ModelName":"%s","InitialInstanceCount":1,"InstanceType":"ml.t2.medium"}]}+ """.formatted(config, model))+ .then().statusCode(200);+ post("SageMaker.CreateEndpoint", """+ {"EndpointName":"sm-reset-ep-%s","EndpointConfigName":"%s"}+ """.formatted(suffix, config))+ .then().statusCode(200).body("EndpointArn", notNullValue());+ } finally {+ // Floci has no DeleteTrainingJob, so a second reset is what leaves nothing behind.+ given().when().post("/_floci/state/reset").then().statusCode(200);+ }+ }++ private Response post(String target, String body) {+ return given().header("Authorization", AUTH)+ .header("X-Amz-Target", target)+ .contentType("application/x-amz-json-1.1")+ .body(body)+ .when().post("/");+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerServiceTest.javaindex 4b5aae4a1c..0138653a9e 100644--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerServiceTest.java@@ -14,6 +14,7 @@ import org.junit.jupiter.api.Test;import java.util.List;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.Mockito.mock;@@ -160,11 +161,51 @@ class SageMakerServiceTest {assertThrows(IllegalArgumentException.class, () -> S3Uri.parse("http://bucket/key"));}+ @Test+ void trainingRunUpdatesApplyOnlyWhileTheirJobIsStillStored() throws Exception {+ InMemoryStorage<String, TrainingJobResource> jobs = new InMemoryStorage<>();+ SageMakerService service = service(jobs);+ String request = """+ {"TrainingJobName":"reset-job",+ "AlgorithmSpecification":{"TrainingImage":"busybox:stable"},+ "OutputDataConfig":{"S3OutputPath":"s3://bucket/out"}}+ """;+ service.createTrainingJob(mapper.readTree(request), "us-east-1");+ TrainingJobResource run = jobs.scan(key -> true).get(0);++ run.trainingJobStatus = "Completed";+ assertTrue(service.updateTrainingJob(run));+ assertEquals("Completed", describeStatus(service, "reset-job"));++ // A state reset wipes the store while the run is still finishing.+ jobs.clear();+ run.trainingJobStatus = "Failed";+ assertFalse(service.updateTrainingJob(run));+ assertTrue(jobs.scan(key -> true).isEmpty());+ assertThrows(AwsException.class, () -> describeStatus(service, "reset-job"));++ // A same-named job created after the reset is not overwritten by the old run.+ service.createTrainingJob(mapper.readTree(request), "us-east-1");+ TrainingJobResource recreated = jobs.scan(key -> true).get(0);+ run.creationTime = recreated.creationTime - 1;+ assertFalse(service.updateTrainingJob(run));+ assertEquals("InProgress", describeStatus(service, "reset-job"));+ }++ private String describeStatus(SageMakerService service, String name) throws Exception {+ return service.describeTrainingJob(mapper.readTree("{\"TrainingJobName\":\"" + name + "\"}"), "us-east-1")+ .path("TrainingJobStatus").asText();+ }+private SageMakerService service() {+ return service(new InMemoryStorage<String, TrainingJobResource>());+ }++ private SageMakerService service(InMemoryStorage<String, TrainingJobResource> trainingJobs) {return new SageMakerService(new InMemoryStorage<String, ModelResource>(),new InMemoryStorage<String, EndpointConfigResource>(),new InMemoryStorage<String, EndpointResource>(),- new InMemoryStorage<String, TrainingJobResource>(),+ trainingJobs,new RegionResolver("us-east-1", "000000000000"), mapper,mock(SageMakerEndpointManager.class), mock(SageMakerTrainingRunner.class));}diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunnerTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunnerTest.javanew file mode 100644index 0000000000..5b60a0bef1--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunnerTest.java@@ -0,0 +1,76 @@+package io.github.hectorvent.floci.services.sagemaker;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;+import org.junit.jupiter.api.Test;++import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.mock;++/**+ * A state reset runs the container teardown, which shuts the worker pool down, and then+ * {@code clear()}. Shutdown runs only the teardown.+ */+class SageMakerTrainingRunnerTest {++ @Test+ void teardownStopsThePoolAndClearRestoresIt() {+ SageMakerTrainingRunner runner = runner();+ assertTrue(runner.acceptsWork());++ runner.stopManagedContainers();+ assertFalse(runner.acceptsWork());++ runner.clear();+ assertTrue(runner.acceptsWork());+ runner.clear();+ assertTrue(runner.acceptsWork());++ runner.stopManagedContainers();+ }++ @Test+ void trainingJobsSubmittedAfterAResetStillRun() throws InterruptedException {+ SageMakerTrainingRunner runner = runner();+ runner.stopManagedContainers();+ runner.clear();++ SageMakerService service = mock(SageMakerService.class);+ CountDownLatch reported = new CountDownLatch(1);+ TrainingJobResource job = new TrainingJobResource();+ job.trainingJobName = "after-reset";+ job.region = "us-east-1";+ job.algorithmSpecification = Map.of("TrainingImage", "busybox:stable");+ // A latch rather than verify(timeout): updateTrainingJob is synchronized, and a timed+ // verify holds the mock's monitor while it polls, blocking the very call it waits for.+ doAnswer(invocation -> {+ reported.countDown();+ return true;+ }).when(service).updateTrainingJob(job);+ runner.runAsync(job, service);++ // The mocked Docker helpers fail the run at once, and every run that executes reports its+ // outcome, so the report proves the new pool accepted and ran the work.+ assertTrue(reported.await(5, TimeUnit.SECONDS));+ runner.stopManagedContainers();+ }++ private static SageMakerTrainingRunner runner() {+ return new SageMakerTrainingRunner(mock(ContainerBuilder.class), mock(ContainerLifecycleManager.class),+ mock(ContainerLogStreamer.class), mock(EmulatorConfig.class, RETURNS_DEEP_STUBS),+ mock(ContainerDetector.class), mock(S3Service.class), new ObjectMapper());+ }+}