Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

test(sagemaker): wait on real conditions and let test containers exit on SIGTERM (#4011)

Priamo Germosen97f64788fba9
diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java
index b9ff351f87..6eeb8ac5c0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java
@@ -12,9 +12,12 @@ import org.junit.jupiter.api.Test;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
+import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
import static io.restassured.RestAssured.given;
+import static org.awaitility.Awaitility.await;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -73,7 +76,7 @@ class SageMakerDockerIntegrationTest {
String model = "sm-ep-model-" + suffix;
String cfg = "sm-ep-cfg-" + suffix;
String endpoint = "sm-endpoint-" + suffix;
- String script = "cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200 if self.path == '/ping' else 404); self.end_headers()\n def do_POST(self):\n n=int(self.headers.get('content-length','0')); b=self.rfile.read(n); self.send_response(200); self.send_header('Content-Type','text/plain'); self.end_headers(); self.wfile.write(b.upper())\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py";
+ String script = "trap 'exit 0' TERM; cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200 if self.path == '/ping' else 404); self.end_headers()\n def do_POST(self):\n n=int(self.headers.get('content-length','0')); b=self.rfile.read(n); self.send_response(200); self.send_header('Content-Type','text/plain'); self.end_headers(); self.wfile.write(b.upper())\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py & wait $!";
post("SageMaker.CreateModel", """
{"ModelName":"%s","PrimaryContainer":{"Image":"public.ecr.aws/docker/library/python:3-alpine","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}
""".formatted(model, json(script))).then().statusCode(200);
@@ -102,24 +105,25 @@ class SageMakerDockerIntegrationTest {
"AlgorithmSpecification":{
"TrainingImage":"public.ecr.aws/docker/library/busybox:stable",
"ContainerEntrypoint":["/bin/sh","-c"],
- "ContainerArguments":["sleep 60"]
+ "ContainerArguments":["trap 'exit 143' TERM; sleep 60 & wait"]
},
"OutputDataConfig":{"S3OutputPath":"s3://%s/output"},
"ResourceConfig":{"InstanceType":"ml.m5.large","InstanceCount":1,"VolumeSizeInGB":1},
"StoppingCondition":{"MaxRuntimeInSeconds":300}
}
""".formatted(job, bucket)).then().statusCode(200);
- // Give the container time to actually be created and start running "sleep 60" so the
- // stop races a real running container rather than one still being staged.
- Thread.sleep(3000);
+ // Wait for the container to actually be running "sleep 60" so the stop races a real
+ // running container rather than one still being staged.
+ awaitContainerRunning("floci-sagemaker-training-" + job);
post("SageMaker.StopTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job)).then().statusCode(200);
post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job))
.then().statusCode(200).body("TrainingJobStatus", equalTo("Stopped"));
- // The runner's exit-code handling races the container disappearing after the stop;
- // give it a beat and confirm it did not relabel the stop a failure once it observes that.
- Thread.sleep(3000);
- post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job))
- .then().statusCode(200).body("TrainingJobStatus", equalTo("Stopped"));
+ // The runner's exit-code handling races the container disappearing after the stop. It
+ // polls the container every 500ms, so keep watching for three of those polls and confirm
+ // it never relabels the stop a failure once it observes the container gone.
+ await().during(Duration.ofMillis(1500)).atMost(Duration.ofSeconds(5)).pollInterval(Duration.ofMillis(100))
+ .untilAsserted(() -> post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job))
+ .then().statusCode(200).body("TrainingJobStatus", equalTo("Stopped")));
}
@Test
@@ -130,7 +134,7 @@ class SageMakerDockerIntegrationTest {
String endpoint = "sm-del-endpoint-" + suffix;
// Sleeps before serving so the /ping health check keeps CreateEndpoint's async start
// in flight well past DeleteEndpoint returning.
- String script = "sleep 5 && cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200); self.end_headers()\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py";
+ String script = "trap 'exit 0' TERM; sleep 5 && cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200); self.end_headers()\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py & wait $!";
post("SageMaker.CreateModel", """
{"ModelName":"%s","PrimaryContainer":{"Image":"public.ecr.aws/docker/library/python:3-alpine","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}
""".formatted(model, json(script))).then().statusCode(200);
@@ -141,10 +145,25 @@ class SageMakerDockerIntegrationTest {
.then().statusCode(200);
post("SageMaker.DeleteEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(200);
// Let the superseded start worker finish (it should discard its result, not persist it).
- Thread.sleep(10000);
+ // Discarding it tears down the container it brought up, so the container appearing and
+ // then disappearing marks the point where the worker has decided.
+ String containerName = "floci-sagemaker-endpoint-" + endpoint;
+ AtomicBoolean containerSeen = new AtomicBoolean();
+ await("the superseded start to discard its container").atMost(Duration.ofSeconds(30))
+ .pollInterval(Duration.ofMillis(100)).until(() -> {
+ boolean running = !dockerClient.listContainersCmd()
+ .withNameFilter(List.of(containerName)).exec().isEmpty();
+ containerSeen.compareAndSet(false, running);
+ return containerSeen.get() && !running;
+ });
post("SageMaker.DescribeEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(400);
}
+ private void awaitContainerRunning(String nameFragment) {
+ await().atMost(Duration.ofSeconds(30)).pollInterval(Duration.ofMillis(100)).until(() ->
+ !dockerClient.listContainersCmd().withNameFilter(List.of(nameFragment)).exec().isEmpty());
+ }
+
private void waitForTraining(String job, String status) throws Exception {
long deadline = System.currentTimeMillis() + Duration.ofSeconds(120).toMillis();
while (System.currentTimeMillis() < deadline) {