Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

test(sagemaker): remove the remaining sleeps from the Docker integration test (#4051)

Priamo Germosenbc9f41df6faf
diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java
index 6eeb8ac5c0..e13e0238f8 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java
@@ -14,6 +14,7 @@ import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
import static io.restassured.RestAssured.given;
@@ -105,15 +106,15 @@ class SageMakerDockerIntegrationTest {
"AlgorithmSpecification":{
"TrainingImage":"public.ecr.aws/docker/library/busybox:stable",
"ContainerEntrypoint":["/bin/sh","-c"],
- "ContainerArguments":["trap 'exit 143' TERM; sleep 60 & wait"]
+ "ContainerArguments":["trap 'exit 143' TERM; mkfifo /tmp/hold; cat /tmp/hold & wait"]
},
"OutputDataConfig":{"S3OutputPath":"s3://%s/output"},
"ResourceConfig":{"InstanceType":"ml.m5.large","InstanceCount":1,"VolumeSizeInGB":1},
"StoppingCondition":{"MaxRuntimeInSeconds":300}
}
""".formatted(job, bucket)).then().statusCode(200);
- // Wait for the container to actually be running "sleep 60" so the stop races a real
- // running container rather than one still being staged.
+ // Wait for the container to actually be running, blocked on an empty pipe, so the stop
+ // races a real running container rather than one still being staged.
awaitContainerRunning("floci-sagemaker-training-" + job);
post("SageMaker.StopTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job)).then().statusCode(200);
post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job))
@@ -132,9 +133,9 @@ class SageMakerDockerIntegrationTest {
String model = "sm-del-model-" + suffix;
String cfg = "sm-del-cfg-" + suffix;
String endpoint = "sm-del-endpoint-" + suffix;
- // Sleeps before serving so the /ping health check keeps CreateEndpoint's async start
- // in flight well past DeleteEndpoint returning.
- String script = "trap 'exit 0' TERM; sleep 5 && cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200); self.end_headers()\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py & wait $!";
+ // DeleteEndpoint returns while CreateEndpoint's async start is still bringing the container
+ // up, well before /ping can succeed, so the start it races is always still in flight.
+ String script = "trap 'exit 0' TERM; cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200); self.end_headers()\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py & wait $!";
post("SageMaker.CreateModel", """
{"ModelName":"%s","PrimaryContainer":{"Image":"public.ecr.aws/docker/library/python:3-alpine","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}
""".formatted(model, json(script))).then().statusCode(200);
@@ -164,26 +165,26 @@ class SageMakerDockerIntegrationTest {
!dockerClient.listContainersCmd().withNameFilter(List.of(nameFragment)).exec().isEmpty());
}
- private void waitForTraining(String job, String status) throws Exception {
- long deadline = System.currentTimeMillis() + Duration.ofSeconds(120).toMillis();
- while (System.currentTimeMillis() < deadline) {
- String current = post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job)).then().statusCode(200).extract().path("TrainingJobStatus");
- if (status.equals(current)) return;
- if ("Failed".equals(current)) throw new AssertionError("Training failed");
- Thread.sleep(1000);
+ private void waitForTraining(String job, String status) {
+ String current = awaitTerminalStatus(status, () -> post("SageMaker.DescribeTrainingJob",
+ "{\"TrainingJobName\":\"%s\"}".formatted(job)).then().statusCode(200).extract().path("TrainingJobStatus"));
+ if ("Failed".equals(current) && !status.equals(current)) {
+ throw new AssertionError("Training failed");
}
- throw new AssertionError("Timed out waiting for training job");
}
- private void waitForEndpoint(String endpoint, String status) throws Exception {
- long deadline = System.currentTimeMillis() + Duration.ofSeconds(150).toMillis();
- while (System.currentTimeMillis() < deadline) {
- String current = post("SageMaker.DescribeEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(200).extract().path("EndpointStatus");
- if (status.equals(current)) return;
- if ("Failed".equals(current)) throw new AssertionError("Endpoint failed");
- Thread.sleep(1000);
+ private void waitForEndpoint(String endpoint, String status) {
+ String current = awaitTerminalStatus(status, () -> post("SageMaker.DescribeEndpoint",
+ "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(200).extract().path("EndpointStatus"));
+ if ("Failed".equals(current) && !status.equals(current)) {
+ throw new AssertionError("Endpoint failed");
}
- throw new AssertionError("Timed out waiting for endpoint");
+ }
+
+ /** Polls until the resource reaches the wanted status, or Failed, and returns the status it reached. */
+ private static String awaitTerminalStatus(String wanted, Callable<String> currentStatus) {
+ return await().atMost(Duration.ofSeconds(150)).pollDelay(Duration.ZERO).pollInterval(Duration.ofMillis(100))
+ .until(currentStatus, current -> wanted.equals(current) || "Failed".equals(current));
}
private io.restassured.response.Response post(String target, String body) {