Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lifecycle): keep SageMaker and ECS background work running after a state reset (#3797)

Hector Ventura08656347c707
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java b/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java
index b0964d15b5..4632be5567 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardown.java
@@ -15,7 +15,9 @@ package io.github.hectorvent.floci.core.common;
* storage flush schedulers are already stopped.
*
* <p>Implementations must be idempotent; they may also be invoked from {@code @PreDestroy}
- * as a fallback.
+ * as a fallback. An implementation that shuts down its own executor here must also implement
+ * {@link Resettable} and replace that executor in {@code clear()}: a reset runs this hook
+ * too, and {@code clear()} runs after it on a reset but never on shutdown.
*/
public interface ContainerTeardown {
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index 0e710c855d..ac37ef3712 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.ContainerTeardown;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageBackedMap;
@@ -63,7 +64,7 @@ import io.github.hectorvent.floci.core.resource.SupportedResourceType;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
@ApplicationScoped
-public class EcsService implements ContainerTeardown, ResourceProvider {
+public class EcsService implements ContainerTeardown, ResourceProvider, Resettable {
private static final Logger LOG = Logger.getLogger(EcsService.class);
private static final String DEFAULT_CLUSTER = "default";
@@ -75,8 +76,9 @@ public class EcsService implements ContainerTeardown, ResourceProvider {
private final EcsEventPublisher eventPublisher;
private final boolean dockerMode;
private final String baseUrl;
- private final ScheduledExecutorService reconciler = Executors.newSingleThreadScheduledExecutor(
- r -> { Thread t = new Thread(r, "ecs-reconciler"); t.setDaemon(true); return t; });
+ // Replaced by clear() after a state reset, whose container teardown shuts this scheduler down.
+ private volatile ScheduledExecutorService reconciler = newReconciler();
+ private final Object reconcilerLock = new Object();
// region::clusterName → EcsCluster
private Map<String, EcsCluster> clusters = new ConcurrentHashMap<>();
@@ -132,7 +134,16 @@ public class EcsService implements ContainerTeardown, ResourceProvider {
@PostConstruct
void init() {
initializeStorage();
- reconciler.scheduleAtFixedRate(this::reconcile, 5, 5, TimeUnit.SECONDS);
+ scheduleReconciliation(reconciler);
+ }
+
+ private void scheduleReconciliation(ScheduledExecutorService scheduler) {
+ scheduler.scheduleAtFixedRate(this::reconcile, 5, 5, TimeUnit.SECONDS);
+ }
+
+ private static ScheduledExecutorService newReconciler() {
+ return Executors.newSingleThreadScheduledExecutor(
+ r -> { Thread t = new Thread(r, "ecs-reconciler"); t.setDaemon(true); return t; });
}
void initializeStorage() {
@@ -191,18 +202,19 @@ public class EcsService implements ContainerTeardown, ResourceProvider {
}
/**
- * Stops the Docker containers of all still-running tasks on emulator shutdown.
- * Task state is transient (memory-only), so without this the containers outlive
- * the process as orphans. The reconciler is shut down first — and any in-flight
- * tick awaited — so it cannot restart drained tasks between this teardown and
- * the final storage flush. Handles are claimed atomically to avoid racing an
- * explicit StopTask.
+ * Stops the Docker containers of all still-running tasks on emulator shutdown and on a
+ * state reset. Task state is transient (memory-only), so without this the containers
+ * outlive the process as orphans. The reconciler is shut down first, and any in-flight
+ * tick awaited, so it cannot restart drained tasks between this teardown and the final
+ * storage flush. A reset brings it back in {@link #clear()}. Handles are claimed
+ * atomically to avoid racing an explicit StopTask.
*/
@Override
public void stopManagedContainers() {
- reconciler.shutdownNow();
+ ScheduledExecutorService current = reconciler;
+ current.shutdownNow();
try {
- reconciler.awaitTermination(2, TimeUnit.SECONDS);
+ current.awaitTermination(2, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
@@ -219,6 +231,22 @@ public class EcsService implements ContainerTeardown, ResourceProvider {
}
}
+ /**
+ * Runs after a state reset has drained the tasks and wiped the store, never on shutdown.
+ * The reset's teardown stopped the reconciler, so without a new one no service would be
+ * reconciled again until the emulator restarted.
+ */
+ @Override
+ public void clear() {
+ synchronized (reconcilerLock) {
+ if (reconciler.isShutdown()) {
+ ScheduledExecutorService replacement = newReconciler();
+ scheduleReconciliation(replacement);
+ reconciler = replacement;
+ }
+ }
+ }
+
boolean isReconcilerShutdown() {
return reconciler.isShutdown();
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java
index ea4c9e2665..581768b1e4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.sagemaker;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.ContainerTeardown;
+import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
@@ -41,7 +42,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@ApplicationScoped
-public class SageMakerEndpointManager implements ContainerTeardown {
+public class SageMakerEndpointManager implements ContainerTeardown, Resettable {
private static final Logger LOG = Logger.getLogger(SageMakerEndpointManager.class);
private static final int PORT = 8080;
private static final int DEFAULT_FILE_MODE = 0644;
@@ -53,7 +54,8 @@ public class SageMakerEndpointManager implements ContainerTeardown {
private final EmulatorConfig config;
private final ContainerDetector containerDetector;
private final S3Service s3Service;
- private final ExecutorService executor = Executors.newCachedThreadPool();
+ // Replaced by clear() after a state reset, whose container teardown shuts this pool down.
+ private volatile ExecutorService executor = Executors.newCachedThreadPool();
private final ConcurrentHashMap<String, String> containers = new ConcurrentHashMap<>();
private final HttpClient httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(2)).build();
@@ -144,12 +146,28 @@ public class SageMakerEndpointManager implements ContainerTeardown {
}
@Override
- public void stopManagedContainers() {
+ public synchronized void stopManagedContainers() {
containers.forEach((name, id) -> lifecycleManager.stopAndRemove(id, null));
containers.clear();
executor.shutdownNow();
}
+ /**
+ * Runs after a state reset has torn the containers down and wiped the store, never on
+ * shutdown. The teardown shut the worker pool down, so without a new one every later
+ * CreateEndpoint and UpdateEndpoint would be rejected until the emulator restarted.
+ */
+ @Override
+ public synchronized void clear() {
+ if (executor.isShutdown()) {
+ executor = Executors.newCachedThreadPool();
+ }
+ }
+
+ boolean acceptsWork() {
+ return !executor.isShutdown();
+ }
+
private List<String> environment(String region, Map<String, String> modelEnv) {
List<String> env = new ArrayList<>();
env.add("SAGEMAKER_BIND_TO_PORT=8080");
diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerService.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerService.java
index 8962b25c64..3aa8c87136 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerService.java
@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.Mod
import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
import java.time.Clock;
import java.util.ArrayList;
@@ -22,12 +23,14 @@ import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.function.BiConsumer;
import java.util.function.Function;
@ApplicationScoped
public class SageMakerService {
+ private static final Logger LOG = Logger.getLogger(SageMakerService.class);
static final int DEFAULT_LIMIT = 100;
static final int MAX_RESULTS_CEILING = 100;
private static final ObjectMapper STATIC_MAPPER = new ObjectMapper();
@@ -468,8 +471,23 @@ public class SageMakerService {
return true;
}
- public synchronized void updateTrainingJob(TrainingJobResource job) {
- trainingJobStore.put(regionKey(job.region, job.trainingJobName), job);
+ /**
+ * Publishes a training run's progress or outcome only while the job the run started from is
+ * still stored. A state reset interrupts in-flight runs and wipes the store, and an
+ * interrupted run's failure write must not bring the wiped job back, whether it lands before
+ * or after the wipe, nor overwrite a same-named job created after the reset.
+ */
+ public synchronized boolean updateTrainingJob(TrainingJobResource job) {
+ String key = regionKey(job.region, job.trainingJobName);
+ TrainingJobResource current = trainingJobStore.get(key).orElse(null);
+ if (current == null || !Objects.equals(current.trainingJobArn, job.trainingJobArn)
+ || current.creationTime != job.creationTime) {
+ LOG.debugv("SageMaker training job {0} is no longer the stored job; discarding its run update",
+ job.trainingJobName);
+ return false;
+ }
+ trainingJobStore.put(key, job);
+ return true;
}
private Map<String, String> resourceTags(String arn) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java
index de6a063964..94e010075a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java
@@ -7,6 +7,7 @@ import com.github.dockerjava.core.command.LogContainerResultCallback;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.ContainerTeardown;
+import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
@@ -42,7 +43,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@ApplicationScoped
-public class SageMakerTrainingRunner implements ContainerTeardown {
+public class SageMakerTrainingRunner implements ContainerTeardown, Resettable {
private static final Logger LOG = Logger.getLogger(SageMakerTrainingRunner.class);
private static final String LOG_GROUP = "/aws/sagemaker/TrainingJobs";
@@ -53,7 +54,8 @@ public class SageMakerTrainingRunner implements ContainerTeardown {
private final ContainerDetector containerDetector;
private final S3Service s3Service;
private final ObjectMapper mapper;
- private final ExecutorService executor = Executors.newCachedThreadPool();
+ // Replaced by clear() after a state reset, whose container teardown shuts this pool down.
+ private volatile ExecutorService executor = Executors.newCachedThreadPool();
private final ConcurrentHashMap<String, String> containers = new ConcurrentHashMap<>();
// Names a stop that stop() has already committed to the store as "Stopped": run()'s exit-code
// handling checks this before persisting, since a stop races the removal of the very container
@@ -146,6 +148,10 @@ public class SageMakerTrainingRunner implements ContainerTeardown {
job.failureReason = e.getMessage();
job.trainingEndTime = System.currentTimeMillis();
service.updateTrainingJob(job);
+ if (e instanceof InterruptedException) {
+ // Interrupted by a teardown's shutdownNow(): keep the flag for the pool thread.
+ Thread.currentThread().interrupt();
+ }
}
}
@@ -158,12 +164,29 @@ public class SageMakerTrainingRunner implements ContainerTeardown {
}
@Override
- public void stopManagedContainers() {
+ public synchronized void stopManagedContainers() {
containers.forEach((name, id) -> lifecycleManager.stopAndRemove(id, null));
containers.clear();
executor.shutdownNow();
}
+ /**
+ * Runs after a state reset has torn the containers down and wiped the store, never on
+ * shutdown. The teardown shut the worker pool down, so without a new one every later
+ * CreateTrainingJob would be rejected until the emulator restarted.
+ */
+ @Override
+ public synchronized void clear() {
+ stopRequested.clear();
+ if (executor.isShutdown()) {
+ executor = Executors.newCachedThreadPool();
+ }
+ }
+
+ boolean acceptsWork() {
+ return !executor.isShutdown();
+ }
+
private List<String> environment(TrainingJobResource job) {
List<String> env = new ArrayList<>();
env.add("TRAINING_JOB_NAME=" + job.trainingJobName);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java
index 454a64e4d0..ef666a60e7 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java
@@ -19,6 +19,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
@@ -76,6 +77,33 @@ class EcsServiceTeardownTest {
verify(containerManager, times(1)).stopTask(handle);
}
+ @Test
+ void clearRestartsTheReconcilerThatAResetTeardownStopped() {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().mock()).thenReturn(false); // docker mode
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ EcsService service = new EcsService(
+ new RegionResolver(REGION, "000000000000"),
+ mock(EcsContainerManager.class),
+ config,
+ mock(EcsLoadBalancerRegistrar.class),
+ new SingleUseStorageFactory(),
+ null);
+ service.initializeStorage();
+
+ // A reset runs the teardown and then clear(); shutdown runs only the teardown.
+ service.stopManagedContainers();
+ assertTrue(service.isReconcilerShutdown());
+
+ service.clear();
+ assertFalse(service.isReconcilerShutdown());
+ service.clear();
+ assertFalse(service.isReconcilerShutdown());
+
+ service.stopManagedContainers();
+ assertTrue(service.isReconcilerShutdown());
+ }
+
@Test
void stoppedTaskRetriesTeardownUntilItsRemainingLogStreamIsReleased() {
EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManagerTest.java
new file mode 100644
index 0000000000..8c5c27d139
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManagerTest.java
@@ -0,0 +1,71 @@
+package io.github.hectorvent.floci.services.sagemaker;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+
+/**
+ * A state reset runs the container teardown, which shuts the worker pool down, and then
+ * {@code clear()}. Shutdown runs only the teardown.
+ */
+class SageMakerEndpointManagerTest {
+
+ @Test
+ void teardownStopsThePoolAndClearRestoresIt() {
+ SageMakerEndpointManager manager = manager();
+ assertTrue(manager.acceptsWork());
+
+ manager.stopManagedContainers();
+ assertFalse(manager.acceptsWork());
+
+ manager.clear();
+ assertTrue(manager.acceptsWork());
+ manager.clear();
+ assertTrue(manager.acceptsWork());
+
+ manager.stopManagedContainers();
+ }
+
+ @Test
+ void endpointStartsSubmittedAfterAResetStillRun() throws InterruptedException {
+ SageMakerEndpointManager manager = manager();
+ manager.stopManagedContainers();
+ manager.clear();
+
+ SageMakerService service = mock(SageMakerService.class);
+ CountDownLatch published = new CountDownLatch(1);
+ EndpointResource endpoint = new EndpointResource();
+ endpoint.endpointName = "after-reset";
+ endpoint.region = "us-east-1";
+ // A latch rather than verify(timeout): finalizeEndpointStart is synchronized, and a timed
+ // verify holds the mock's monitor while it polls, blocking the very call it waits for.
+ doAnswer(invocation -> {
+ published.countDown();
+ return true;
+ }).when(service).finalizeEndpointStart(endpoint);
+ manager.startEndpointAsync(endpoint, service);
+
+ // The mocked service has no model for the endpoint, so the start fails at once, and every
+ // start that executes publishes its outcome: the publish proves the new pool ran the work.
+ assertTrue(published.await(5, TimeUnit.SECONDS));
+ manager.stopManagedContainers();
+ }
+
+ private static SageMakerEndpointManager manager() {
+ return new SageMakerEndpointManager(mock(ContainerBuilder.class), mock(ContainerLifecycleManager.class),
+ mock(EmulatorConfig.class, RETURNS_DEEP_STUBS), mock(ContainerDetector.class), mock(S3Service.class));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerResetIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerResetIntegrationTest.java
new file mode 100644
index 0000000000..eea9aa351b
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerResetIntegrationTest.java
@@ -0,0 +1,70 @@
+package io.github.hectorvent.floci.services.sagemaker;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.notNullValue;
+
+/**
+ * A state reset runs every container teardown, and the SageMaker training runner and endpoint
+ * manager shut their worker pools down there. Without restoring them afterwards, every later
+ * CreateTrainingJob, CreateEndpoint and UpdateEndpoint failed with a 500 until the emulator
+ * restarted. The requests submit their work before touching Docker, so no daemon is needed.
+ */
+@QuarkusTest
+class SageMakerResetIntegrationTest {
+ private static final String AUTH =
+ "AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/sagemaker/aws4_request";
+ private static final String UNPULLABLE_IMAGE = "floci.invalid/none:latest";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void trainingJobsAndEndpointsCanStillBeCreatedAfterAStateReset() {
+ given().when().post("/_floci/state/reset").then().statusCode(200);
+ try {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ post("SageMaker.CreateTrainingJob", """
+ {"TrainingJobName":"sm-reset-%s",
+ "AlgorithmSpecification":{"TrainingImage":"%s","TrainingInputMode":"File"},
+ "OutputDataConfig":{"S3OutputPath":"s3://sm-reset-%s/output"},
+ "ResourceConfig":{"InstanceType":"ml.m5.large","InstanceCount":1,"VolumeSizeInGB":1},
+ "StoppingCondition":{"MaxRuntimeInSeconds":60}}
+ """.formatted(suffix, UNPULLABLE_IMAGE, suffix))
+ .then().statusCode(200).body("TrainingJobArn", notNullValue());
+
+ String model = "sm-reset-model-" + suffix;
+ post("SageMaker.CreateModel", """
+ {"ModelName":"%s","PrimaryContainer":{"Image":"%s"}}
+ """.formatted(model, UNPULLABLE_IMAGE))
+ .then().statusCode(200);
+ String config = "sm-reset-cfg-" + suffix;
+ post("SageMaker.CreateEndpointConfig", """
+ {"EndpointConfigName":"%s","ProductionVariants":[{"VariantName":"AllTraffic","ModelName":"%s","InitialInstanceCount":1,"InstanceType":"ml.t2.medium"}]}
+ """.formatted(config, model))
+ .then().statusCode(200);
+ post("SageMaker.CreateEndpoint", """
+ {"EndpointName":"sm-reset-ep-%s","EndpointConfigName":"%s"}
+ """.formatted(suffix, config))
+ .then().statusCode(200).body("EndpointArn", notNullValue());
+ } finally {
+ // Floci has no DeleteTrainingJob, so a second reset is what leaves nothing behind.
+ given().when().post("/_floci/state/reset").then().statusCode(200);
+ }
+ }
+
+ private Response post(String target, String body) {
+ return given().header("Authorization", AUTH)
+ .header("X-Amz-Target", target)
+ .contentType("application/x-amz-json-1.1")
+ .body(body)
+ .when().post("/");
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerServiceTest.java
index 4b5aae4a1c..0138653a9e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerServiceTest.java
@@ -14,6 +14,7 @@ import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
@@ -160,11 +161,51 @@ class SageMakerServiceTest {
assertThrows(IllegalArgumentException.class, () -> S3Uri.parse("http://bucket/key"));
}
+ @Test
+ void trainingRunUpdatesApplyOnlyWhileTheirJobIsStillStored() throws Exception {
+ InMemoryStorage<String, TrainingJobResource> jobs = new InMemoryStorage<>();
+ SageMakerService service = service(jobs);
+ String request = """
+ {"TrainingJobName":"reset-job",
+ "AlgorithmSpecification":{"TrainingImage":"busybox:stable"},
+ "OutputDataConfig":{"S3OutputPath":"s3://bucket/out"}}
+ """;
+ service.createTrainingJob(mapper.readTree(request), "us-east-1");
+ TrainingJobResource run = jobs.scan(key -> true).get(0);
+
+ run.trainingJobStatus = "Completed";
+ assertTrue(service.updateTrainingJob(run));
+ assertEquals("Completed", describeStatus(service, "reset-job"));
+
+ // A state reset wipes the store while the run is still finishing.
+ jobs.clear();
+ run.trainingJobStatus = "Failed";
+ assertFalse(service.updateTrainingJob(run));
+ assertTrue(jobs.scan(key -> true).isEmpty());
+ assertThrows(AwsException.class, () -> describeStatus(service, "reset-job"));
+
+ // A same-named job created after the reset is not overwritten by the old run.
+ service.createTrainingJob(mapper.readTree(request), "us-east-1");
+ TrainingJobResource recreated = jobs.scan(key -> true).get(0);
+ run.creationTime = recreated.creationTime - 1;
+ assertFalse(service.updateTrainingJob(run));
+ assertEquals("InProgress", describeStatus(service, "reset-job"));
+ }
+
+ private String describeStatus(SageMakerService service, String name) throws Exception {
+ return service.describeTrainingJob(mapper.readTree("{\"TrainingJobName\":\"" + name + "\"}"), "us-east-1")
+ .path("TrainingJobStatus").asText();
+ }
+
private SageMakerService service() {
+ return service(new InMemoryStorage<String, TrainingJobResource>());
+ }
+
+ private SageMakerService service(InMemoryStorage<String, TrainingJobResource> trainingJobs) {
return new SageMakerService(new InMemoryStorage<String, ModelResource>(),
new InMemoryStorage<String, EndpointConfigResource>(),
new InMemoryStorage<String, EndpointResource>(),
- new InMemoryStorage<String, TrainingJobResource>(),
+ trainingJobs,
new RegionResolver("us-east-1", "000000000000"), mapper,
mock(SageMakerEndpointManager.class), mock(SageMakerTrainingRunner.class));
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunnerTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunnerTest.java
new file mode 100644
index 0000000000..5b60a0bef1
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunnerTest.java
@@ -0,0 +1,76 @@
+package io.github.hectorvent.floci.services.sagemaker;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+
+/**
+ * A state reset runs the container teardown, which shuts the worker pool down, and then
+ * {@code clear()}. Shutdown runs only the teardown.
+ */
+class SageMakerTrainingRunnerTest {
+
+ @Test
+ void teardownStopsThePoolAndClearRestoresIt() {
+ SageMakerTrainingRunner runner = runner();
+ assertTrue(runner.acceptsWork());
+
+ runner.stopManagedContainers();
+ assertFalse(runner.acceptsWork());
+
+ runner.clear();
+ assertTrue(runner.acceptsWork());
+ runner.clear();
+ assertTrue(runner.acceptsWork());
+
+ runner.stopManagedContainers();
+ }
+
+ @Test
+ void trainingJobsSubmittedAfterAResetStillRun() throws InterruptedException {
+ SageMakerTrainingRunner runner = runner();
+ runner.stopManagedContainers();
+ runner.clear();
+
+ SageMakerService service = mock(SageMakerService.class);
+ CountDownLatch reported = new CountDownLatch(1);
+ TrainingJobResource job = new TrainingJobResource();
+ job.trainingJobName = "after-reset";
+ job.region = "us-east-1";
+ job.algorithmSpecification = Map.of("TrainingImage", "busybox:stable");
+ // A latch rather than verify(timeout): updateTrainingJob is synchronized, and a timed
+ // verify holds the mock's monitor while it polls, blocking the very call it waits for.
+ doAnswer(invocation -> {
+ reported.countDown();
+ return true;
+ }).when(service).updateTrainingJob(job);
+ runner.runAsync(job, service);
+
+ // The mocked Docker helpers fail the run at once, and every run that executes reports its
+ // outcome, so the report proves the new pool accepted and ran the work.
+ assertTrue(reported.await(5, TimeUnit.SECONDS));
+ runner.stopManagedContainers();
+ }
+
+ private static SageMakerTrainingRunner runner() {
+ return new SageMakerTrainingRunner(mock(ContainerBuilder.class), mock(ContainerLifecycleManager.class),
+ mock(ContainerLogStreamer.class), mock(EmulatorConfig.class, RETURNS_DEEP_STUBS),
+ mock(ContainerDetector.class), mock(S3Service.class), new ObjectMapper());
+ }
+}