Light, fluffy, and always free - The AWS Local Emulator alternative
test(sagemaker): wait on real conditions and let test containers exit on SIGTERM (#4011)
Priamo Germosen97f64788fba9
diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.javaindex b9ff351f87..6eeb8ac5c0 100644--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java@@ -12,9 +12,12 @@ import org.junit.jupiter.api.Test;import java.nio.charset.StandardCharsets;import java.time.Duration;+import java.util.List;import java.util.Map;+import java.util.concurrent.atomic.AtomicBoolean;import static io.restassured.RestAssured.given;+import static org.awaitility.Awaitility.await;import static org.hamcrest.Matchers.equalTo;import static org.hamcrest.Matchers.notNullValue;import static org.junit.jupiter.api.Assertions.assertNotNull;@@ -73,7 +76,7 @@ class SageMakerDockerIntegrationTest {String model = "sm-ep-model-" + suffix;String cfg = "sm-ep-cfg-" + suffix;String endpoint = "sm-endpoint-" + suffix;- String script = "cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200 if self.path == '/ping' else 404); self.end_headers()\n def do_POST(self):\n n=int(self.headers.get('content-length','0')); b=self.rfile.read(n); self.send_response(200); self.send_header('Content-Type','text/plain'); self.end_headers(); self.wfile.write(b.upper())\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py";+ String script = "trap 'exit 0' TERM; cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200 if self.path == '/ping' else 404); self.end_headers()\n def do_POST(self):\n n=int(self.headers.get('content-length','0')); b=self.rfile.read(n); self.send_response(200); self.send_header('Content-Type','text/plain'); self.end_headers(); self.wfile.write(b.upper())\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py & wait $!";post("SageMaker.CreateModel", """{"ModelName":"%s","PrimaryContainer":{"Image":"public.ecr.aws/docker/library/python:3-alpine","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}""".formatted(model, json(script))).then().statusCode(200);@@ -102,24 +105,25 @@ class SageMakerDockerIntegrationTest {"AlgorithmSpecification":{"TrainingImage":"public.ecr.aws/docker/library/busybox:stable","ContainerEntrypoint":["/bin/sh","-c"],- "ContainerArguments":["sleep 60"]+ "ContainerArguments":["trap 'exit 143' TERM; sleep 60 & wait"]},"OutputDataConfig":{"S3OutputPath":"s3://%s/output"},"ResourceConfig":{"InstanceType":"ml.m5.large","InstanceCount":1,"VolumeSizeInGB":1},"StoppingCondition":{"MaxRuntimeInSeconds":300}}""".formatted(job, bucket)).then().statusCode(200);- // Give the container time to actually be created and start running "sleep 60" so the- // stop races a real running container rather than one still being staged.- Thread.sleep(3000);+ // Wait for the container to actually be running "sleep 60" so the stop races a real+ // running container rather than one still being staged.+ awaitContainerRunning("floci-sagemaker-training-" + job);post("SageMaker.StopTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job)).then().statusCode(200);post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job)).then().statusCode(200).body("TrainingJobStatus", equalTo("Stopped"));- // The runner's exit-code handling races the container disappearing after the stop;- // give it a beat and confirm it did not relabel the stop a failure once it observes that.- Thread.sleep(3000);- post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job))- .then().statusCode(200).body("TrainingJobStatus", equalTo("Stopped"));+ // The runner's exit-code handling races the container disappearing after the stop. It+ // polls the container every 500ms, so keep watching for three of those polls and confirm+ // it never relabels the stop a failure once it observes the container gone.+ await().during(Duration.ofMillis(1500)).atMost(Duration.ofSeconds(5)).pollInterval(Duration.ofMillis(100))+ .untilAsserted(() -> post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job))+ .then().statusCode(200).body("TrainingJobStatus", equalTo("Stopped")));}@Test@@ -130,7 +134,7 @@ class SageMakerDockerIntegrationTest {String endpoint = "sm-del-endpoint-" + suffix;// Sleeps before serving so the /ping health check keeps CreateEndpoint's async start// in flight well past DeleteEndpoint returning.- String script = "sleep 5 && cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200); self.end_headers()\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py";+ String script = "trap 'exit 0' TERM; sleep 5 && cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200); self.end_headers()\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py & wait $!";post("SageMaker.CreateModel", """{"ModelName":"%s","PrimaryContainer":{"Image":"public.ecr.aws/docker/library/python:3-alpine","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}""".formatted(model, json(script))).then().statusCode(200);@@ -141,10 +145,25 @@ class SageMakerDockerIntegrationTest {.then().statusCode(200);post("SageMaker.DeleteEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(200);// Let the superseded start worker finish (it should discard its result, not persist it).- Thread.sleep(10000);+ // Discarding it tears down the container it brought up, so the container appearing and+ // then disappearing marks the point where the worker has decided.+ String containerName = "floci-sagemaker-endpoint-" + endpoint;+ AtomicBoolean containerSeen = new AtomicBoolean();+ await("the superseded start to discard its container").atMost(Duration.ofSeconds(30))+ .pollInterval(Duration.ofMillis(100)).until(() -> {+ boolean running = !dockerClient.listContainersCmd()+ .withNameFilter(List.of(containerName)).exec().isEmpty();+ containerSeen.compareAndSet(false, running);+ return containerSeen.get() && !running;+ });post("SageMaker.DescribeEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(400);}+ private void awaitContainerRunning(String nameFragment) {+ await().atMost(Duration.ofSeconds(30)).pollInterval(Duration.ofMillis(100)).until(() ->+ !dockerClient.listContainersCmd().withNameFilter(List.of(nameFragment)).exec().isEmpty());+ }+private void waitForTraining(String job, String status) throws Exception {long deadline = System.currentTimeMillis() + Duration.ofSeconds(120).toMillis();while (System.currentTimeMillis() < deadline) {