Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(sagemaker): retry model and training archive uploads (#5999)

Aniket Shukla2b45288e1391
diff --git a/docs/configuration/docker.md b/docs/configuration/docker.md
index e2cad1a143..db98f7acd2 100644
--- a/docs/configuration/docker.md
+++ b/docs/configuration/docker.md
@@ -324,7 +324,8 @@ A request is only replayed when doing so cannot change semantics: requests carry
Archive uploads that use the shared tar-copy helper retry above the transport layer, rebuilding
the upload stream for each attempt with the same six-attempt backoff. This includes user-data
-scripts, the container CA bundle, Verdaccio's configuration, and EKS audit policies, IRSA
+scripts, the container CA bundle, Verdaccio's configuration, SageMaker model data and training
+inputs, and EKS audit policies, IRSA
signing keys, the containerd certificate-directory link and the Pod Identity manifest. Only the
upload is retried, not script execution. Daemon rejections still stop immediately, and each
caller keeps its existing failure handling after retries are exhausted.
diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java
index 6b57d35db6..7c2112d17d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java
@@ -11,6 +11,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.C
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;
import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
+import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;
@@ -70,10 +71,14 @@ public class SageMakerEndpointManager implements ContainerTeardown, Resettable {
}
public void startEndpointAsync(EndpointResource endpoint, SageMakerService service) {
- executor.submit(() -> startEndpoint(endpoint, service));
+ startEndpointAsync(endpoint, service, RetryingTarCopier.DEFAULT_BACKOFF_MS);
}
- private void startEndpoint(EndpointResource endpoint, SageMakerService service) {
+ void startEndpointAsync(EndpointResource endpoint, SageMakerService service, long backoffMillis) {
+ executor.submit(() -> startEndpoint(endpoint, service, backoffMillis));
+ }
+
+ private void startEndpoint(EndpointResource endpoint, SageMakerService service, long backoffMillis) {
String containerId = null;
try {
ModelResource model = service.modelForEndpoint(endpoint);
@@ -97,7 +102,7 @@ public class SageMakerEndpointManager implements ContainerTeardown, Resettable {
containerId = lifecycleManager.create(spec);
String modelDataUrl = string(c.get("ModelDataUrl"));
if (modelDataUrl != null && !modelDataUrl.isBlank()) {
- copyModelData(containerId, modelDataUrl);
+ copyModelData(containerId, modelDataUrl, backoffMillis);
}
ContainerInfo info = lifecycleManager.startCreated(containerId, spec);
EndpointInfo ep = info.getEndpoint(PORT);
@@ -210,14 +215,13 @@ public class SageMakerEndpointManager implements ContainerTeardown, Resettable {
}
}
- private void copyModelData(String containerId, String s3Uri) throws Exception {
+ void copyModelData(String containerId, String s3Uri, long backoffMillis) throws Exception {
S3Uri uri = S3Uri.parse(s3Uri);
S3Object object = s3Service.getObject(uri.bucket(), uri.key());
byte[] tarData = tarModelData(object.getData(), uri.key());
- lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)
- .withRemotePath("/")
- .withTarInputStream(new ByteArrayInputStream(tarData))
- .exec();
+ RetryingTarCopier.copyStreamed(lifecycleManager.getDockerClient(), containerId, "/",
+ "sagemaker-model-data", out -> out.write(tarData),
+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, backoffMillis);
}
private byte[] tarModelData(byte[] data, String key) throws Exception {
diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java
index be79d78fed..d719a4fe85 100644
--- a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java
@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
+import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;
@@ -26,7 +27,6 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
import org.jboss.logging.Logger;
-import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.InputStream;
@@ -80,10 +80,14 @@ public class SageMakerTrainingRunner implements ContainerTeardown, Resettable {
}
public void runAsync(TrainingJobResource job, SageMakerService service) {
- executor.submit(() -> run(job, service));
+ runAsync(job, service, RetryingTarCopier.DEFAULT_BACKOFF_MS);
}
- private void run(TrainingJobResource job, SageMakerService service) {
+ void runAsync(TrainingJobResource job, SageMakerService service, long backoffMillis) {
+ executor.submit(() -> run(job, service, backoffMillis));
+ }
+
+ private void run(TrainingJobResource job, SageMakerService service, long backoffMillis) {
String containerId = null;
Closeable logs = null;
try {
@@ -106,7 +110,7 @@ public class SageMakerTrainingRunner implements ContainerTeardown, Resettable {
containerId = lifecycleManager.create(spec);
containers.put(job.trainingJobName, containerId);
job.containerId = containerId;
- copyTrainingInput(containerId, job);
+ copyTrainingInput(containerId, job, backoffMillis);
lifecycleManager.startCreated(containerId, spec);
job.trainingStartTime = System.currentTimeMillis();
job.secondaryStatus = "Training";
@@ -220,7 +224,7 @@ public class SageMakerTrainingRunner implements ContainerTeardown, Resettable {
return containerDetector.isRunningInContainer() ? config.hostname().orElse(EmbeddedDnsServer.DEFAULT_SUFFIX) : "host.docker.internal";
}
- private void copyTrainingInput(String containerId, TrainingJobResource job) throws Exception {
+ void copyTrainingInput(String containerId, TrainingJobResource job, long backoffMillis) throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
try (TarArchiveOutputStream tar = new TarArchiveOutputStream(out)) {
SageMakerEndpointManager.addFile(tar, "opt/ml/input/config/hyperparameters.json",
@@ -233,10 +237,10 @@ public class SageMakerTrainingRunner implements ContainerTeardown, Resettable {
copyChannelObjects(tar, channel);
}
}
- lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)
- .withRemotePath("/")
- .withTarInputStream(new ByteArrayInputStream(out.toByteArray()))
- .exec();
+ byte[] tarData = out.toByteArray();
+ RetryingTarCopier.copyStreamed(lifecycleManager.getDockerClient(), containerId, "/",
+ "sagemaker-training-input", stream -> stream.write(tarData),
+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, backoffMillis);
}
private Map<String, Object> inputDataConfig(List<Map<String, Object>> channels) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerArchiveRetryTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerArchiveRetryTest.java
new file mode 100644
index 0000000000..1a217547a1
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerArchiveRetryTest.java
@@ -0,0 +1,304 @@
+package io.github.hectorvent.floci.services.sagemaker;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;
+import com.github.dockerjava.api.exception.NotFoundException;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.ModelResource;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.net.SocketException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.RETURNS_SELF;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class SageMakerArchiveRetryTest {
+ private final DockerClient docker = mock(DockerClient.class);
+ private final ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);
+ private final S3Service s3 = mock(S3Service.class);
+ private final ObjectMapper mapper = new ObjectMapper();
+ private final List<InputStream> streams = new ArrayList<>();
+ private final List<CopyArchiveToContainerCmd> commands = new ArrayList<>();
+ private final List<byte[]> completed = new ArrayList<>();
+ private final byte[] payload = "model-data".repeat(2000).getBytes(StandardCharsets.UTF_8);
+ private SageMakerEndpointManager manager;
+ private SageMakerTrainingRunner runner;
+ private TrainingJobResource job;
+ private int failures;
+ private int consumedBytes;
+ private RuntimeException failure;
+
+ @BeforeEach
+ void setUp() {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().dockerNetwork()).thenReturn(Optional.empty());
+ when(config.services().sagemaker().dockerNetwork()).thenReturn(Optional.empty());
+ when(config.docker().resourceNamespace()).thenReturn(Optional.empty());
+ when(config.docker().logMaxSize()).thenReturn("10m");
+ when(config.docker().logMaxFile()).thenReturn("3");
+ ContainerBuilder builder = new ContainerBuilder(config,
+ mock(DockerHostResolver.class), mock(EmbeddedDnsServer.class));
+ manager = new SageMakerEndpointManager(builder, lifecycle, config, mock(ContainerDetector.class), s3);
+ runner = new SageMakerTrainingRunner(builder, lifecycle, mock(ContainerLogStreamer.class), config,
+ mock(ContainerDetector.class), s3, mapper, mock(SageMakerGpuResolver.class));
+ job = new TrainingJobResource();
+ job.trainingJobName = "retry-training";
+ job.region = "us-east-1";
+ job.algorithmSpecification = Map.of("TrainingImage", "busybox:stable");
+ job.hyperParameters = Map.of("epochs", "3");
+ job.inputDataConfig = List.of(Map.of("ChannelName", "train", "DataSource",
+ Map.of("S3DataSource", Map.of("S3Uri", "s3://bucket/input/"))));
+ S3Object input = new S3Object("bucket", "input/nested/data.csv", payload, "text/plain");
+ when(s3.getObject("bucket", "model.bin"))
+ .thenReturn(new S3Object("bucket", "model.bin", payload, "application/octet-stream"));
+ when(s3.listObjects("bucket", "input/", null, 1000)).thenReturn(List.of(input));
+ when(s3.getObject("bucket", "input/nested/data.csv")).thenReturn(input);
+ when(lifecycle.create(any(ContainerSpec.class))).thenReturn("container");
+ when(lifecycle.getDockerClient()).thenReturn(docker);
+ when(docker.copyArchiveToContainerCmd("container")).thenAnswer(ignored -> {
+ CopyArchiveToContainerCmd command = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);
+ commands.add(command);
+ AtomicReference<InputStream> inputStream = new AtomicReference<>();
+ when(command.withTarInputStream(any(InputStream.class))).thenAnswer(invocation -> {
+ InputStream stream = invocation.getArgument(0);
+ streams.add(stream);
+ inputStream.set(stream);
+ return command;
+ });
+ doAnswer(invocation -> {
+ if (commands.size() <= failures) {
+ inputStream.get().readNBytes(consumedBytes);
+ throw failure;
+ }
+ completed.add(inputStream.get().readAllBytes());
+ return null;
+ }).when(command).exec();
+ return command;
+ });
+ }
+
+ @AfterEach
+ void stopWorkers() {
+ manager.stopManagedContainers();
+ runner.stopManagedContainers();
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {0, 73, 8192})
+ void modelUploadRetriesWithTheCompleteArchive(int consumed) throws Exception {
+ failTransport(consumed);
+
+ manager.copyModelData("container", "s3://bucket/model.bin", 0);
+
+ assertRetried();
+ assertArchive(Map.of("opt/ml/model/model.bin", payload));
+ verify(s3).getObject("bucket", "model.bin");
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {0, 73, 8192})
+ void trainingUploadRetriesWithAllConfigurationAndChannelFiles(int consumed) throws Exception {
+ failTransport(consumed);
+
+ runner.copyTrainingInput("container", job, 0);
+
+ assertRetried();
+ assertArchive(Map.of(
+ "opt/ml/input/config/hyperparameters.json", mapper.writeValueAsBytes(job.hyperParameters),
+ "opt/ml/input/config/resourceconfig.json",
+ "{\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\"]}".getBytes(StandardCharsets.UTF_8),
+ "opt/ml/input/config/inputdataconfig.json",
+ "{\"train\":{\"ContentType\":\"application/octet-stream\",\"TrainingInputMode\":\"File\"}}"
+ .getBytes(StandardCharsets.UTF_8),
+ "opt/ml/input/data/train/nested/data.csv", payload));
+ verify(s3).listObjects("bucket", "input/", null, 1000);
+ verify(s3).getObject("bucket", "input/nested/data.csv");
+ }
+
+ @Test
+ void compressedModelRetryPreservesNestedPathsAndRejectsTraversal() throws Exception {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (TarArchiveOutputStream tar = new TarArchiveOutputStream(new GzipCompressorOutputStream(out))) {
+ SageMakerEndpointManager.addFile(tar, "nested/model.bin", payload);
+ SageMakerEndpointManager.addFile(tar, "../escaped.bin", payload);
+ }
+ when(s3.getObject("bucket", "model.tar.gz"))
+ .thenReturn(new S3Object("bucket", "model.tar.gz", out.toByteArray(), "application/gzip"));
+ failTransport(73);
+
+ manager.copyModelData("container", "s3://bucket/model.tar.gz", 0);
+
+ assertRetried();
+ assertArchive(Map.of("opt/ml/model/nested/model.bin", payload));
+ verify(s3).getObject("bucket", "model.tar.gz");
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void exhaustedTransportFailureStopsAtSixAttempts(boolean training) {
+ failTransport(73);
+ failures = 10;
+
+ RuntimeException thrown = assertThrows(RuntimeException.class, () -> copy(training));
+
+ assertEquals(6, commands.size());
+ assertSame(failure, thrown.getCause());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void daemonRejectionDoesNotRetry(boolean training) {
+ failures = 10;
+ failure = new NotFoundException("container missing");
+
+ RuntimeException thrown = assertThrows(RuntimeException.class, () -> copy(training));
+
+ assertEquals(1, commands.size());
+ assertTrue(thrown.getMessage().contains("container missing"));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void sourceFailureDoesNotReachDocker(boolean training) {
+ when(s3.getObject("bucket", "model.bin")).thenThrow(new IllegalArgumentException("source unavailable"));
+ when(s3.getObject("bucket", "input/nested/data.csv"))
+ .thenThrow(new IllegalArgumentException("source unavailable"));
+
+ IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, () -> copy(training));
+
+ assertEquals("source unavailable", thrown.getMessage());
+ assertTrue(commands.isEmpty());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void failedUploadRemovesTheUnstartedContainer(boolean training) throws InterruptedException {
+ failures = 1;
+ failure = new NotFoundException("container missing");
+ assertFailedStart(training, 1, "container missing");
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void exhaustedUploadRetriesRemoveTheUnstartedContainer(boolean training) throws InterruptedException {
+ failTransport(73);
+ failures = 10;
+ assertFailedStart(training, 6, "Connection reset");
+ }
+
+ private void assertFailedStart(boolean training, int attempts, String reason) throws InterruptedException {
+ SageMakerService service = mock(SageMakerService.class);
+ CountDownLatch reported = new CountDownLatch(1);
+ EndpointResource endpoint = new EndpointResource();
+ endpoint.endpointName = "retry-endpoint";
+ endpoint.region = "us-east-1";
+ ModelResource model = new ModelResource();
+ model.primaryContainer = Map.of("Image", "busybox:stable", "ModelDataUrl", "s3://bucket/model.bin");
+ when(service.modelForEndpoint(endpoint)).thenReturn(model);
+ doAnswer(invocation -> {
+ reported.countDown();
+ return true;
+ }).when(service).finalizeEndpointStart(endpoint);
+ doAnswer(invocation -> {
+ reported.countDown();
+ return true;
+ }).when(service).updateTrainingJob(job);
+
+ if (training) {
+ runner.runAsync(job, service, 0);
+ } else {
+ manager.startEndpointAsync(endpoint, service, 0);
+ }
+
+ assertTrue(reported.await(5, TimeUnit.SECONDS));
+ assertEquals("Failed", training ? job.trainingJobStatus : endpoint.endpointStatus);
+ assertTrue((training ? job.failureReason : endpoint.failureReason).contains(reason));
+ assertEquals(attempts, commands.size());
+ verify(lifecycle).create(any(ContainerSpec.class));
+ verify(lifecycle).stopAndRemove("container", null);
+ verify(lifecycle, never()).startCreated(any(), any());
+ }
+
+ private void copy(boolean training) throws Exception {
+ if (training) {
+ runner.copyTrainingInput("container", job, 0);
+ } else {
+ manager.copyModelData("container", "s3://bucket/model.bin", 0);
+ }
+ }
+
+ private void failTransport(int consumed) {
+ failures = 1;
+ consumedBytes = consumed;
+ failure = new RuntimeException(new SocketException("Connection reset"));
+ }
+
+ private void assertRetried() {
+ assertEquals(2, commands.size());
+ assertNotSame(commands.get(0), commands.get(1));
+ assertNotSame(streams.get(0), streams.get(1));
+ for (CopyArchiveToContainerCmd command : commands) {
+ verify(command).withRemotePath("/");
+ }
+ assertEquals(1, completed.size());
+ }
+
+ private void assertArchive(Map<String, byte[]> expected) throws Exception {
+ Map<String, byte[]> actual = new LinkedHashMap<>();
+ try (TarArchiveInputStream tar = new TarArchiveInputStream(new ByteArrayInputStream(completed.getFirst()))) {
+ TarArchiveEntry entry;
+ while ((entry = tar.getNextEntry()) != null) {
+ assertEquals(0644, entry.getMode());
+ actual.put(entry.getName(), tar.readAllBytes());
+ }
+ }
+ assertEquals(expected.keySet(), actual.keySet());
+ for (String name : expected.keySet()) {
+ assertArrayEquals(expected.get(name), actual.get(name), name);
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java
index d8ecfa11ca..fa16206225 100644
--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java
@@ -85,13 +85,16 @@ class SageMakerDockerIntegrationTest {
@Test
void endpointHostsRealContainerAndRuntimeProxiesInvocations() throws Exception {
String suffix = Long.toString(System.nanoTime(), 36);
+ String bucket = "sm-ep-" + suffix;
+ s3Service.createBucket(bucket, "us-east-1");
+ s3Service.putObject(bucket, "prefix.txt", "MODEL:".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
String model = "sm-ep-model-" + suffix;
String cfg = "sm-ep-cfg-" + suffix;
String endpoint = "sm-endpoint-" + suffix;
- String script = "trap 'exit 0' TERM; cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200 if self.path == '/ping' else 404); self.end_headers()\n def do_POST(self):\n n=int(self.headers.get('content-length','0')); b=self.rfile.read(n); self.send_response(200); self.send_header('Content-Type','text/plain'); self.end_headers(); self.wfile.write(b.upper())\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py & wait $!";
+ String script = "trap 'exit 0' TERM; cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nfrom pathlib import Path\nmodel=Path('/opt/ml/model/prefix.txt').read_bytes()\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200 if self.path == '/ping' else 404); self.end_headers()\n def do_POST(self):\n n=int(self.headers.get('content-length','0')); b=self.rfile.read(n); self.send_response(200); self.send_header('Content-Type','text/plain'); self.end_headers(); self.wfile.write(model+b.upper())\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py & wait $!";
post("SageMaker.CreateModel", """
- {"ModelName":"%s","PrimaryContainer":{"Image":"%s","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}
- """.formatted(model, TestImages.PYTHON_ALPINE, json(script))).then().statusCode(200);
+ {"ModelName":"%s","PrimaryContainer":{"Image":"%s","ModelDataUrl":"s3://%s/prefix.txt","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}
+ """.formatted(model, TestImages.PYTHON_ALPINE, bucket, json(script))).then().statusCode(200);
post("SageMaker.CreateEndpointConfig", """
{"EndpointConfigName":"%s","ProductionVariants":[{"VariantName":"AllTraffic","ModelName":"%s","InitialInstanceCount":1,"InstanceType":"ml.t2.medium","InitialVariantWeight":1.0}]}
""".formatted(cfg, model)).then().statusCode(200);
@@ -101,7 +104,7 @@ class SageMakerDockerIntegrationTest {
given().header("Authorization", "AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/runtime.sagemaker/aws4_request")
.contentType("text/plain").body("hello")
.when().post("/endpoints/%s/invocations".formatted(endpoint))
- .then().statusCode(200).body(equalTo("HELLO"));
+ .then().statusCode(200).body(equalTo("MODEL:HELLO"));
post("SageMaker.DeleteEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(200);
}