Light, fluffy, and always free - The AWS Local Emulator alternative
fix(sagemaker): retry model and training archive uploads (#5999)
Aniket Shukla2b45288e1391
diff --git a/docs/configuration/docker.md b/docs/configuration/docker.mdindex e2cad1a143..db98f7acd2 100644--- a/docs/configuration/docker.md+++ b/docs/configuration/docker.md@@ -324,7 +324,8 @@ A request is only replayed when doing so cannot change semantics: requests carryArchive uploads that use the shared tar-copy helper retry above the transport layer, rebuildingthe upload stream for each attempt with the same six-attempt backoff. This includes user-data-scripts, the container CA bundle, Verdaccio's configuration, and EKS audit policies, IRSA+scripts, the container CA bundle, Verdaccio's configuration, SageMaker model data and training+inputs, and EKS audit policies, IRSAsigning keys, the containerd certificate-directory link and the Pod Identity manifest. Only theupload is retried, not script execution. Daemon rejections still stop immediately, and eachcaller keeps its existing failure handling after retries are exhausted.diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.javaindex 6b57d35db6..7c2112d17d 100644--- a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java@@ -11,6 +11,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.Cimport io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;+import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.s3.model.S3Object;import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;@@ -70,10 +71,14 @@ public class SageMakerEndpointManager implements ContainerTeardown, Resettable {}public void startEndpointAsync(EndpointResource endpoint, SageMakerService service) {- executor.submit(() -> startEndpoint(endpoint, service));+ startEndpointAsync(endpoint, service, RetryingTarCopier.DEFAULT_BACKOFF_MS);}- private void startEndpoint(EndpointResource endpoint, SageMakerService service) {+ void startEndpointAsync(EndpointResource endpoint, SageMakerService service, long backoffMillis) {+ executor.submit(() -> startEndpoint(endpoint, service, backoffMillis));+ }++ private void startEndpoint(EndpointResource endpoint, SageMakerService service, long backoffMillis) {String containerId = null;try {ModelResource model = service.modelForEndpoint(endpoint);@@ -97,7 +102,7 @@ public class SageMakerEndpointManager implements ContainerTeardown, Resettable {containerId = lifecycleManager.create(spec);String modelDataUrl = string(c.get("ModelDataUrl"));if (modelDataUrl != null && !modelDataUrl.isBlank()) {- copyModelData(containerId, modelDataUrl);+ copyModelData(containerId, modelDataUrl, backoffMillis);}ContainerInfo info = lifecycleManager.startCreated(containerId, spec);EndpointInfo ep = info.getEndpoint(PORT);@@ -210,14 +215,13 @@ public class SageMakerEndpointManager implements ContainerTeardown, Resettable {}}- private void copyModelData(String containerId, String s3Uri) throws Exception {+ void copyModelData(String containerId, String s3Uri, long backoffMillis) throws Exception {S3Uri uri = S3Uri.parse(s3Uri);S3Object object = s3Service.getObject(uri.bucket(), uri.key());byte[] tarData = tarModelData(object.getData(), uri.key());- lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)- .withRemotePath("/")- .withTarInputStream(new ByteArrayInputStream(tarData))- .exec();+ RetryingTarCopier.copyStreamed(lifecycleManager.getDockerClient(), containerId, "/",+ "sagemaker-model-data", out -> out.write(tarData),+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, backoffMillis);}private byte[] tarModelData(byte[] data, String key) throws Exception {diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.javaindex be79d78fed..d719a4fe85 100644--- a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;+import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.s3.model.S3Object;import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;@@ -26,7 +27,6 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;import org.jboss.logging.Logger;-import java.io.ByteArrayInputStream;import java.io.ByteArrayOutputStream;import java.io.Closeable;import java.io.InputStream;@@ -80,10 +80,14 @@ public class SageMakerTrainingRunner implements ContainerTeardown, Resettable {}public void runAsync(TrainingJobResource job, SageMakerService service) {- executor.submit(() -> run(job, service));+ runAsync(job, service, RetryingTarCopier.DEFAULT_BACKOFF_MS);}- private void run(TrainingJobResource job, SageMakerService service) {+ void runAsync(TrainingJobResource job, SageMakerService service, long backoffMillis) {+ executor.submit(() -> run(job, service, backoffMillis));+ }++ private void run(TrainingJobResource job, SageMakerService service, long backoffMillis) {String containerId = null;Closeable logs = null;try {@@ -106,7 +110,7 @@ public class SageMakerTrainingRunner implements ContainerTeardown, Resettable {containerId = lifecycleManager.create(spec);containers.put(job.trainingJobName, containerId);job.containerId = containerId;- copyTrainingInput(containerId, job);+ copyTrainingInput(containerId, job, backoffMillis);lifecycleManager.startCreated(containerId, spec);job.trainingStartTime = System.currentTimeMillis();job.secondaryStatus = "Training";@@ -220,7 +224,7 @@ public class SageMakerTrainingRunner implements ContainerTeardown, Resettable {return containerDetector.isRunningInContainer() ? config.hostname().orElse(EmbeddedDnsServer.DEFAULT_SUFFIX) : "host.docker.internal";}- private void copyTrainingInput(String containerId, TrainingJobResource job) throws Exception {+ void copyTrainingInput(String containerId, TrainingJobResource job, long backoffMillis) throws Exception {ByteArrayOutputStream out = new ByteArrayOutputStream();try (TarArchiveOutputStream tar = new TarArchiveOutputStream(out)) {SageMakerEndpointManager.addFile(tar, "opt/ml/input/config/hyperparameters.json",@@ -233,10 +237,10 @@ public class SageMakerTrainingRunner implements ContainerTeardown, Resettable {copyChannelObjects(tar, channel);}}- lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)- .withRemotePath("/")- .withTarInputStream(new ByteArrayInputStream(out.toByteArray()))- .exec();+ byte[] tarData = out.toByteArray();+ RetryingTarCopier.copyStreamed(lifecycleManager.getDockerClient(), containerId, "/",+ "sagemaker-training-input", stream -> stream.write(tarData),+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, backoffMillis);}private Map<String, Object> inputDataConfig(List<Map<String, Object>> channels) {diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerArchiveRetryTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerArchiveRetryTest.javanew file mode 100644index 0000000000..1a217547a1--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerArchiveRetryTest.java@@ -0,0 +1,304 @@+package io.github.hectorvent.floci.services.sagemaker;++import com.fasterxml.jackson.databind.ObjectMapper;+import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;+import com.github.dockerjava.api.exception.NotFoundException;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.ModelResource;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.io.ByteArrayInputStream;+import java.io.ByteArrayOutputStream;+import java.io.InputStream;+import java.net.SocketException;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.Optional;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;++import static org.junit.jupiter.api.Assertions.assertArrayEquals;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotSame;+import static org.junit.jupiter.api.Assertions.assertSame;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.RETURNS_SELF;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++class SageMakerArchiveRetryTest {+ private final DockerClient docker = mock(DockerClient.class);+ private final ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);+ private final S3Service s3 = mock(S3Service.class);+ private final ObjectMapper mapper = new ObjectMapper();+ private final List<InputStream> streams = new ArrayList<>();+ private final List<CopyArchiveToContainerCmd> commands = new ArrayList<>();+ private final List<byte[]> completed = new ArrayList<>();+ private final byte[] payload = "model-data".repeat(2000).getBytes(StandardCharsets.UTF_8);+ private SageMakerEndpointManager manager;+ private SageMakerTrainingRunner runner;+ private TrainingJobResource job;+ private int failures;+ private int consumedBytes;+ private RuntimeException failure;++ @BeforeEach+ void setUp() {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().dockerNetwork()).thenReturn(Optional.empty());+ when(config.services().sagemaker().dockerNetwork()).thenReturn(Optional.empty());+ when(config.docker().resourceNamespace()).thenReturn(Optional.empty());+ when(config.docker().logMaxSize()).thenReturn("10m");+ when(config.docker().logMaxFile()).thenReturn("3");+ ContainerBuilder builder = new ContainerBuilder(config,+ mock(DockerHostResolver.class), mock(EmbeddedDnsServer.class));+ manager = new SageMakerEndpointManager(builder, lifecycle, config, mock(ContainerDetector.class), s3);+ runner = new SageMakerTrainingRunner(builder, lifecycle, mock(ContainerLogStreamer.class), config,+ mock(ContainerDetector.class), s3, mapper, mock(SageMakerGpuResolver.class));+ job = new TrainingJobResource();+ job.trainingJobName = "retry-training";+ job.region = "us-east-1";+ job.algorithmSpecification = Map.of("TrainingImage", "busybox:stable");+ job.hyperParameters = Map.of("epochs", "3");+ job.inputDataConfig = List.of(Map.of("ChannelName", "train", "DataSource",+ Map.of("S3DataSource", Map.of("S3Uri", "s3://bucket/input/"))));+ S3Object input = new S3Object("bucket", "input/nested/data.csv", payload, "text/plain");+ when(s3.getObject("bucket", "model.bin"))+ .thenReturn(new S3Object("bucket", "model.bin", payload, "application/octet-stream"));+ when(s3.listObjects("bucket", "input/", null, 1000)).thenReturn(List.of(input));+ when(s3.getObject("bucket", "input/nested/data.csv")).thenReturn(input);+ when(lifecycle.create(any(ContainerSpec.class))).thenReturn("container");+ when(lifecycle.getDockerClient()).thenReturn(docker);+ when(docker.copyArchiveToContainerCmd("container")).thenAnswer(ignored -> {+ CopyArchiveToContainerCmd command = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);+ commands.add(command);+ AtomicReference<InputStream> inputStream = new AtomicReference<>();+ when(command.withTarInputStream(any(InputStream.class))).thenAnswer(invocation -> {+ InputStream stream = invocation.getArgument(0);+ streams.add(stream);+ inputStream.set(stream);+ return command;+ });+ doAnswer(invocation -> {+ if (commands.size() <= failures) {+ inputStream.get().readNBytes(consumedBytes);+ throw failure;+ }+ completed.add(inputStream.get().readAllBytes());+ return null;+ }).when(command).exec();+ return command;+ });+ }++ @AfterEach+ void stopWorkers() {+ manager.stopManagedContainers();+ runner.stopManagedContainers();+ }++ @ParameterizedTest+ @ValueSource(ints = {0, 73, 8192})+ void modelUploadRetriesWithTheCompleteArchive(int consumed) throws Exception {+ failTransport(consumed);++ manager.copyModelData("container", "s3://bucket/model.bin", 0);++ assertRetried();+ assertArchive(Map.of("opt/ml/model/model.bin", payload));+ verify(s3).getObject("bucket", "model.bin");+ }++ @ParameterizedTest+ @ValueSource(ints = {0, 73, 8192})+ void trainingUploadRetriesWithAllConfigurationAndChannelFiles(int consumed) throws Exception {+ failTransport(consumed);++ runner.copyTrainingInput("container", job, 0);++ assertRetried();+ assertArchive(Map.of(+ "opt/ml/input/config/hyperparameters.json", mapper.writeValueAsBytes(job.hyperParameters),+ "opt/ml/input/config/resourceconfig.json",+ "{\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\"]}".getBytes(StandardCharsets.UTF_8),+ "opt/ml/input/config/inputdataconfig.json",+ "{\"train\":{\"ContentType\":\"application/octet-stream\",\"TrainingInputMode\":\"File\"}}"+ .getBytes(StandardCharsets.UTF_8),+ "opt/ml/input/data/train/nested/data.csv", payload));+ verify(s3).listObjects("bucket", "input/", null, 1000);+ verify(s3).getObject("bucket", "input/nested/data.csv");+ }++ @Test+ void compressedModelRetryPreservesNestedPathsAndRejectsTraversal() throws Exception {+ ByteArrayOutputStream out = new ByteArrayOutputStream();+ try (TarArchiveOutputStream tar = new TarArchiveOutputStream(new GzipCompressorOutputStream(out))) {+ SageMakerEndpointManager.addFile(tar, "nested/model.bin", payload);+ SageMakerEndpointManager.addFile(tar, "../escaped.bin", payload);+ }+ when(s3.getObject("bucket", "model.tar.gz"))+ .thenReturn(new S3Object("bucket", "model.tar.gz", out.toByteArray(), "application/gzip"));+ failTransport(73);++ manager.copyModelData("container", "s3://bucket/model.tar.gz", 0);++ assertRetried();+ assertArchive(Map.of("opt/ml/model/nested/model.bin", payload));+ verify(s3).getObject("bucket", "model.tar.gz");+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void exhaustedTransportFailureStopsAtSixAttempts(boolean training) {+ failTransport(73);+ failures = 10;++ RuntimeException thrown = assertThrows(RuntimeException.class, () -> copy(training));++ assertEquals(6, commands.size());+ assertSame(failure, thrown.getCause());+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void daemonRejectionDoesNotRetry(boolean training) {+ failures = 10;+ failure = new NotFoundException("container missing");++ RuntimeException thrown = assertThrows(RuntimeException.class, () -> copy(training));++ assertEquals(1, commands.size());+ assertTrue(thrown.getMessage().contains("container missing"));+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void sourceFailureDoesNotReachDocker(boolean training) {+ when(s3.getObject("bucket", "model.bin")).thenThrow(new IllegalArgumentException("source unavailable"));+ when(s3.getObject("bucket", "input/nested/data.csv"))+ .thenThrow(new IllegalArgumentException("source unavailable"));++ IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, () -> copy(training));++ assertEquals("source unavailable", thrown.getMessage());+ assertTrue(commands.isEmpty());+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void failedUploadRemovesTheUnstartedContainer(boolean training) throws InterruptedException {+ failures = 1;+ failure = new NotFoundException("container missing");+ assertFailedStart(training, 1, "container missing");+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void exhaustedUploadRetriesRemoveTheUnstartedContainer(boolean training) throws InterruptedException {+ failTransport(73);+ failures = 10;+ assertFailedStart(training, 6, "Connection reset");+ }++ private void assertFailedStart(boolean training, int attempts, String reason) throws InterruptedException {+ SageMakerService service = mock(SageMakerService.class);+ CountDownLatch reported = new CountDownLatch(1);+ EndpointResource endpoint = new EndpointResource();+ endpoint.endpointName = "retry-endpoint";+ endpoint.region = "us-east-1";+ ModelResource model = new ModelResource();+ model.primaryContainer = Map.of("Image", "busybox:stable", "ModelDataUrl", "s3://bucket/model.bin");+ when(service.modelForEndpoint(endpoint)).thenReturn(model);+ doAnswer(invocation -> {+ reported.countDown();+ return true;+ }).when(service).finalizeEndpointStart(endpoint);+ doAnswer(invocation -> {+ reported.countDown();+ return true;+ }).when(service).updateTrainingJob(job);++ if (training) {+ runner.runAsync(job, service, 0);+ } else {+ manager.startEndpointAsync(endpoint, service, 0);+ }++ assertTrue(reported.await(5, TimeUnit.SECONDS));+ assertEquals("Failed", training ? job.trainingJobStatus : endpoint.endpointStatus);+ assertTrue((training ? job.failureReason : endpoint.failureReason).contains(reason));+ assertEquals(attempts, commands.size());+ verify(lifecycle).create(any(ContainerSpec.class));+ verify(lifecycle).stopAndRemove("container", null);+ verify(lifecycle, never()).startCreated(any(), any());+ }++ private void copy(boolean training) throws Exception {+ if (training) {+ runner.copyTrainingInput("container", job, 0);+ } else {+ manager.copyModelData("container", "s3://bucket/model.bin", 0);+ }+ }++ private void failTransport(int consumed) {+ failures = 1;+ consumedBytes = consumed;+ failure = new RuntimeException(new SocketException("Connection reset"));+ }++ private void assertRetried() {+ assertEquals(2, commands.size());+ assertNotSame(commands.get(0), commands.get(1));+ assertNotSame(streams.get(0), streams.get(1));+ for (CopyArchiveToContainerCmd command : commands) {+ verify(command).withRemotePath("/");+ }+ assertEquals(1, completed.size());+ }++ private void assertArchive(Map<String, byte[]> expected) throws Exception {+ Map<String, byte[]> actual = new LinkedHashMap<>();+ try (TarArchiveInputStream tar = new TarArchiveInputStream(new ByteArrayInputStream(completed.getFirst()))) {+ TarArchiveEntry entry;+ while ((entry = tar.getNextEntry()) != null) {+ assertEquals(0644, entry.getMode());+ actual.put(entry.getName(), tar.readAllBytes());+ }+ }+ assertEquals(expected.keySet(), actual.keySet());+ for (String name : expected.keySet()) {+ assertArrayEquals(expected.get(name), actual.get(name), name);+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.javaindex d8ecfa11ca..fa16206225 100644--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java@@ -85,13 +85,16 @@ class SageMakerDockerIntegrationTest {@Testvoid endpointHostsRealContainerAndRuntimeProxiesInvocations() throws Exception {String suffix = Long.toString(System.nanoTime(), 36);+ String bucket = "sm-ep-" + suffix;+ s3Service.createBucket(bucket, "us-east-1");+ s3Service.putObject(bucket, "prefix.txt", "MODEL:".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());String model = "sm-ep-model-" + suffix;String cfg = "sm-ep-cfg-" + suffix;String endpoint = "sm-endpoint-" + suffix;- String script = "trap 'exit 0' TERM; cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200 if self.path == '/ping' else 404); self.end_headers()\n def do_POST(self):\n n=int(self.headers.get('content-length','0')); b=self.rfile.read(n); self.send_response(200); self.send_header('Content-Type','text/plain'); self.end_headers(); self.wfile.write(b.upper())\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py & wait $!";+ String script = "trap 'exit 0' TERM; cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nfrom pathlib import Path\nmodel=Path('/opt/ml/model/prefix.txt').read_bytes()\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200 if self.path == '/ping' else 404); self.end_headers()\n def do_POST(self):\n n=int(self.headers.get('content-length','0')); b=self.rfile.read(n); self.send_response(200); self.send_header('Content-Type','text/plain'); self.end_headers(); self.wfile.write(model+b.upper())\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py & wait $!";post("SageMaker.CreateModel", """- {"ModelName":"%s","PrimaryContainer":{"Image":"%s","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}- """.formatted(model, TestImages.PYTHON_ALPINE, json(script))).then().statusCode(200);+ {"ModelName":"%s","PrimaryContainer":{"Image":"%s","ModelDataUrl":"s3://%s/prefix.txt","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}+ """.formatted(model, TestImages.PYTHON_ALPINE, bucket, json(script))).then().statusCode(200);post("SageMaker.CreateEndpointConfig", """{"EndpointConfigName":"%s","ProductionVariants":[{"VariantName":"AllTraffic","ModelName":"%s","InitialInstanceCount":1,"InstanceType":"ml.t2.medium","InitialVariantWeight":1.0}]}""".formatted(cfg, model)).then().statusCode(200);@@ -101,7 +104,7 @@ class SageMakerDockerIntegrationTest {given().header("Authorization", "AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/runtime.sagemaker/aws4_request").contentType("text/plain").body("hello").when().post("/endpoints/%s/invocations".formatted(endpoint))- .then().statusCode(200).body(equalTo("HELLO"));+ .then().statusCode(200).body(equalTo("MODEL:HELLO"));post("SageMaker.DeleteEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(200);}