Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(sagemaker): add SageMaker emulation with Docker-backed training (#1949)

Marcus Bergoe2aefa654bbf
diff --git a/compatibility-tests/sdk-test-python/tests/test_sagemaker.py b/compatibility-tests/sdk-test-python/tests/test_sagemaker.py
new file mode 100644
index 0000000000..2c3cc99965
--- /dev/null
+++ b/compatibility-tests/sdk-test-python/tests/test_sagemaker.py
@@ -0,0 +1,74 @@
+import os
+import time
+import uuid
+
+import boto3
+import pytest
+from botocore.config import Config
+
+
+def client(service):
+ return boto3.client(
+ service,
+ # Every other test file gets this from the conftest.py fixtures, which read
+ # FLOCI_ENDPOINT (the test runner container's compat-net hostname for floci, e.g.
+ # http://floci:4566). This file has no fixture of its own, so it must read the same
+ # variable directly instead of hardcoding localhost, which only resolves to floci
+ # when running the suite straight on the host rather than in its own CI container.
+ endpoint_url=os.environ.get("FLOCI_ENDPOINT", "http://localhost:4566"),
+ region_name="us-east-1",
+ aws_access_key_id="test",
+ aws_secret_access_key="test",
+ config=Config(retries={"max_attempts": 1}),
+ )
+
+
+def test_sagemaker_control_plane_and_training():
+ sm = client("sagemaker")
+ s3 = client("s3")
+ suffix = uuid.uuid4().hex[:8]
+ model = f"compat-sm-{suffix}"
+ cfg = f"compat-sm-cfg-{suffix}"
+ bucket = f"compat-sm-{suffix}"
+
+ sm.create_model(ModelName=model, PrimaryContainer={"Image": "public.ecr.aws/docker/library/busybox:stable"}, ExecutionRoleArn="arn:aws:iam::000000000000:role/r")
+ assert sm.describe_model(ModelName=model)["ModelName"] == model
+ sm.create_endpoint_config(EndpointConfigName=cfg, ProductionVariants=[{"VariantName": "AllTraffic", "ModelName": model, "InitialInstanceCount": 1, "InstanceType": "ml.t2.medium"}])
+ assert any(c["EndpointConfigName"] == cfg for c in sm.list_endpoint_configs()["EndpointConfigs"])
+
+ s3.create_bucket(Bucket=bucket)
+ s3.put_object(Bucket=bucket, Key="input/data.txt", Body=b"hello")
+ job = f"compat-train-{suffix}"
+ sm.create_training_job(
+ TrainingJobName=job,
+ RoleArn="arn:aws:iam::000000000000:role/r",
+ AlgorithmSpecification={
+ "TrainingImage": "public.ecr.aws/docker/library/busybox:stable",
+ "TrainingInputMode": "File",
+ "ContainerEntrypoint": ["/bin/sh", "-c"],
+ "ContainerArguments": ["mkdir -p /opt/ml/model && echo ok > /opt/ml/model/model.txt"],
+ },
+ InputDataConfig=[{
+ "ChannelName": "train",
+ "DataSource": {"S3DataSource": {"S3DataType": "S3Prefix", "S3Uri": f"s3://{bucket}/input"}},
+ }],
+ OutputDataConfig={"S3OutputPath": f"s3://{bucket}/output"},
+ ResourceConfig={"InstanceType": "ml.m5.large", "InstanceCount": 1, "VolumeSizeInGB": 1},
+ StoppingCondition={"MaxRuntimeInSeconds": 60},
+ )
+ deadline = time.time() + 120
+ while time.time() < deadline:
+ desc = sm.describe_training_job(TrainingJobName=job)
+ if desc["TrainingJobStatus"] == "Completed":
+ artifact = desc["ModelArtifacts"]["S3ModelArtifacts"]
+ key = artifact.split(f"s3://{bucket}/", 1)[1]
+ assert s3.get_object(Bucket=bucket, Key=key)["Body"].read()
+ break
+ if desc["TrainingJobStatus"] == "Failed":
+ pytest.fail(desc.get("FailureReason", "training failed"))
+ time.sleep(1)
+ else:
+ pytest.fail("Timed out waiting for SageMaker training job")
+
+ sm.delete_endpoint_config(EndpointConfigName=cfg)
+ sm.delete_model(ModelName=model)
diff --git a/docs/services/index.md b/docs/services/index.md
index 1a80506a4d..557a92c61f 100644
--- a/docs/services/index.md
+++ b/docs/services/index.md
@@ -120,6 +120,8 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac
| [CloudFront](cloudfront.md) | `/2020-05-31/distribution/*`, `/2020-05-31/cache-policy/*`, `/2020-05-31/function/*` | REST XML | 50 |
| [Route53](route53.md) | `/2013-04-01/hostedzone/*`, `/2013-04-01/healthcheck/*`, `/2013-04-01/change/*` | REST XML | 25 |
| [Route 53 Resolver](route53resolver.md) | `POST /` + `X-Amz-Target: Route53Resolver.*` | JSON 1.1 | 18 |
+| [Amazon SageMaker](sagemaker.md) | `POST /` + `X-Amz-Target: SageMaker.*` | JSON 1.1 | 20 |
+| [SageMaker Runtime](sagemaker.md#endpoint-hosting) | `/endpoints/{EndpointName}/invocations` | REST (binary payload) | 1 |
| [Cloud Map](cloudmap.md) | `POST /` + `X-Amz-Target: Route53AutoNaming_v20170314.*` | JSON 1.1 | 22 |
| [AWS Config](config.md) | `POST /` + `X-Amz-Target: StarlingDoveService.*` | JSON 1.1 | 33 |
| [CloudTrail](cloudtrail.md) | `POST /` + `X-Amz-Target: com.amazonaws.cloudtrail.v20131101.CloudTrail_20131101.*` | JSON 1.1 | 9 |
@@ -154,3 +156,5 @@ export AWS_SECRET_ACCESS_KEY=test
```
`AWS_ENDPOINT_URL` is the standard env var recognised by the AWS CLI v2 and AWS SDKs v2+, so no `--endpoint-url` flag is needed on each command.
+
+- [SageMaker](sagemaker.md) - model, endpoint, runtime, and training emulation with Docker execution.
diff --git a/docs/services/sagemaker.md b/docs/services/sagemaker.md
new file mode 100644
index 0000000000..ddba8dd45a
--- /dev/null
+++ b/docs/services/sagemaker.md
@@ -0,0 +1,63 @@
+# Amazon SageMaker
+
+Floci emulates SageMaker control-plane APIs and runs real Docker containers for local training jobs and hosted endpoints. Containers use the SageMaker `/opt/ml` contract rather than mocks.
+
+## Supported operations
+
+| Area | Operations |
+| --- | --- |
+| Models | `CreateModel`, `DescribeModel`, `DeleteModel`, `ListModels` |
+| Endpoint configs | `CreateEndpointConfig`, `DescribeEndpointConfig`, `DeleteEndpointConfig`, `ListEndpointConfigs` |
+| Endpoints | `CreateEndpoint`, `DescribeEndpoint`, `UpdateEndpoint`, `DeleteEndpoint`, `ListEndpoints` |
+| Training | `CreateTrainingJob`, `DescribeTrainingJob`, `ListTrainingJobs`, `StopTrainingJob` |
+| Tags | `AddTags`, `ListTags`, `DeleteTags` |
+| Runtime | `POST /endpoints/{EndpointName}/invocations` |
+
+## Supported Actions
+
+<!-- floci:actions:start -->
+| Action | Description |
+| --- | --- |
+| `CreateModel` | Registers a model from `PrimaryContainer` or `Containers`; rejects an empty `Containers` array rather than dereferencing it. |
+| `DescribeModel` | Returns a model by name, scoped to the calling region. |
+| `DeleteModel` | Deletes a model by name, scoped to the calling region. |
+| `ListModels` | Lists models for the calling region; supports `NameContains`, `MaxResults` (1-100), and `NextToken` pagination. |
+| `CreateEndpointConfig` | Creates an endpoint configuration; every `ProductionVariants[].ModelName` must already exist. |
+| `DescribeEndpointConfig` | Returns an endpoint configuration by name, scoped to the calling region. |
+| `DeleteEndpointConfig` | Deletes an endpoint configuration by name, scoped to the calling region. |
+| `ListEndpointConfigs` | Lists endpoint configurations for the calling region; supports `MaxResults` and `NextToken`. |
+| `CreateEndpoint` | Starts hosting the configured model image asynchronously; the endpoint is `Creating` until the container passes its `/ping` check. |
+| `DescribeEndpoint` | Returns an endpoint's status, scoped to the calling region. |
+| `DeleteEndpoint` | Stops and removes the endpoint's container; a start still in flight for this endpoint discards its result instead of resurrecting the deleted record. |
+| `ListEndpoints` | Lists endpoints for the calling region; supports `MaxResults` and `NextToken`. |
+| `UpdateEndpoint` | Swaps an endpoint to a new endpoint configuration, restarting its container; a superseded in-flight start discards its result. |
+| `CreateTrainingJob` | Runs `AlgorithmSpecification.TrainingImage` asynchronously against the `/opt/ml` contract and uploads the resulting model artifacts to S3. |
+| `DescribeTrainingJob` | Returns a training job's status, scoped to the calling region. |
+| `ListTrainingJobs` | Lists training jobs for the calling region; supports `NameContains`, `StatusEquals`, `MaxResults`, and `NextToken`. |
+| `StopTrainingJob` | Stops a training job that is `InProgress` or `Stopping`; a no-op against a job already in a terminal state. |
+| `AddTags` | Adds tags to a model, endpoint config, endpoint, or training job identified by ARN. |
+| `ListTags` | Lists tags for a model, endpoint config, endpoint, or training job identified by ARN. |
+| `DeleteTags` | Removes tags by key from a model, endpoint config, endpoint, or training job identified by ARN. |
+<!-- floci:actions:end -->
+
+## Training contract
+
+`CreateTrainingJob` starts `AlgorithmSpecification.TrainingImage` with command `train` unless `ContainerEntrypoint`/`ContainerArguments` are supplied. Floci writes SageMaker config files under `/opt/ml/input/config`, downloads channel data from S3 into `/opt/ml/input/data/<channel>`, waits for container exit, and uploads `/opt/ml/model` as `model.tar.gz` under `OutputDataConfig.S3OutputPath/<TrainingJobName>/output/`.
+
+## Endpoint hosting
+
+`CreateEndpoint` starts the model image as a long-lived Docker container with command `serve`, port `8080`, `/ping` health checks, and `/invocations` runtime proxying. `ModelDataUrl` artifacts are downloaded from S3 and placed in `/opt/ml/model`.
+
+## Examples
+
+```python
+import boto3
+sm = boto3.client("sagemaker", endpoint_url="http://localhost:4566", region_name="us-east-1")
+sm.create_model(ModelName="m", PrimaryContainer={"Image":"my-image"}, ExecutionRoleArn="arn:aws:iam::000000000000:role/r")
+sm.create_endpoint_config(EndpointConfigName="cfg", ProductionVariants=[{"VariantName":"AllTraffic","ModelName":"m","InitialInstanceCount":1,"InstanceType":"ml.t2.medium"}])
+sm.create_endpoint(EndpointName="ep", EndpointConfigName="cfg")
+```
+
+```bash
+aws --endpoint-url=http://localhost:4566 sagemaker list-models
+```
diff --git a/mkdocs.yml b/mkdocs.yml
index 03e3a04106..4e7a6be03c 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -181,6 +181,7 @@ nav:
- Elastic Beanstalk: services/elastic-beanstalk.md
- CodeBuild: services/codebuild.md
- AWS Batch: services/batch.md
+ - SageMaker: services/sagemaker.md
- CodeDeploy: services/codedeploy.md
- CodeGuru Reviewer: services/codegurureviewer.md
- CodePipeline: services/codepipeline.md
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 0911a51c7d..3f2602d139 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -336,6 +336,7 @@ public interface EmulatorConfig {
LakeFormationStorageConfig lakeformation();
EfsStorageConfig efs();
+ SageMakerStorageConfig sagemaker();
}
interface ApsStorageConfig {
@@ -611,6 +612,7 @@ public interface EmulatorConfig {
@WithDefault("5000")
long flushIntervalMs();
}
+
interface EfsStorageConfig {
Optional<String> mode();
@@ -618,6 +620,13 @@ public interface EmulatorConfig {
long flushIntervalMs();
}
+ interface SageMakerStorageConfig {
+ Optional<String> mode();
+
+ @WithDefault("5000")
+ long flushIntervalMs();
+ }
+
interface CodeDeployStorageConfig {
Optional<String> mode();
@@ -727,6 +736,7 @@ public interface EmulatorConfig {
CloudFrontServiceConfig cloudfront();
AppSyncServiceConfig appsync();
BatchServiceConfig batch();
+ SageMakerServiceConfig sagemaker();
LightsailServiceConfig lightsail();
UiServiceConfig ui();
S3VectorsServiceConfig s3vectors();
@@ -1104,6 +1114,13 @@ public interface EmulatorConfig {
Optional<String> dockerNetwork();
}
+ interface SageMakerServiceConfig {
+ @WithDefault("true")
+ boolean enabled();
+
+ Optional<String> dockerNetwork();
+ }
+
interface CodeDeployServiceConfig {
@WithDefault("true")
boolean enabled();
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java
index 3ad4b3b93f..b943824fc0 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java
@@ -57,6 +57,7 @@ import io.github.hectorvent.floci.services.route53resolver.Route53ResolverJsonHa
import io.github.hectorvent.floci.services.networkfirewall.NetworkFirewallJsonHandler;
import io.github.hectorvent.floci.services.servicecatalog.ServiceCatalogJsonHandler;
import io.github.hectorvent.floci.services.servicequotas.ServiceQuotasJsonHandler;
+import io.github.hectorvent.floci.services.sagemaker.SageMakerJsonHandler;
import io.github.hectorvent.floci.services.ssm.Ec2MessagesJsonHandler;
import io.github.hectorvent.floci.services.ssm.SsmJsonHandler;
import jakarta.inject.Inject;
@@ -139,6 +140,7 @@ public class AwsJson11Controller {
private final ServiceQuotasJsonHandler serviceQuotasJsonHandler;
private final MarketplaceEntitlementController marketplaceEntitlementController;
private final MarketplaceMeteringController marketplaceMeteringController;
+ private final SageMakerJsonHandler sageMakerJsonHandler;
@Inject
public AwsJson11Controller(ObjectMapper objectMapper, ResolvedServiceCatalog catalog,
@@ -192,7 +194,8 @@ public class AwsJson11Controller {
BudgetsJsonHandler budgetsJsonHandler,
ServiceQuotasJsonHandler serviceQuotasJsonHandler,
MarketplaceEntitlementController marketplaceEntitlementController,
- MarketplaceMeteringController marketplaceMeteringController) {
+ MarketplaceMeteringController marketplaceMeteringController,
+ SageMakerJsonHandler sageMakerJsonHandler) {
this.objectMapper = objectMapper;
this.strictBodyReader = objectMapper.reader().with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
this.catalog = catalog;
@@ -251,6 +254,7 @@ public class AwsJson11Controller {
this.serviceQuotasJsonHandler = serviceQuotasJsonHandler;
this.marketplaceEntitlementController = marketplaceEntitlementController;
this.marketplaceMeteringController = marketplaceMeteringController;
+ this.sageMakerJsonHandler = sageMakerJsonHandler;
}
@POST
@@ -362,6 +366,7 @@ public class AwsJson11Controller {
}
yield marketplaceMeteringController.handle(action, request, region);
}
+ case "sagemaker" -> sageMakerJsonHandler.handle(action, request, region);
default -> null;
};
// catalog.matchTarget is protocol-agnostic: a JSON 1.0 target
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
index 53e6968784..32999c1e03 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
@@ -60,6 +60,7 @@ import io.github.hectorvent.floci.services.marketplace.MarketplaceCatalogControl
import io.github.hectorvent.floci.services.marketplace.MarketplaceDeploymentController;
import io.github.hectorvent.floci.services.marketplace.MarketplaceDiscoveryController;
import io.github.hectorvent.floci.services.marketplace.MarketplaceReportingController;
+import io.github.hectorvent.floci.services.sagemaker.SageMakerRuntimeController;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
@@ -402,6 +403,12 @@ public class ResolvedServiceCatalog {
config.storage().services().batch().flushIntervalMs(), null, ServiceProtocol.REST_JSON,
protocols(ServiceProtocol.REST_JSON),
Set.of(), Set.of("batch"), Set.of(), Set.of(BatchController.class)),
+ descriptor("sagemaker", "sagemaker", config.services().sagemaker().enabled(), true,
+ "sagemaker", storageMode(config.storage().services().sagemaker().mode(), config.storage().mode()),
+ config.storage().services().sagemaker().flushIntervalMs(), null, ServiceProtocol.JSON,
+ protocols(ServiceProtocol.JSON, ServiceProtocol.REST_JSON),
+ Set.of("SageMaker."), Set.of("sagemaker", "runtime.sagemaker"), Set.of(),
+ Set.of(SageMakerRuntimeController.class)),
descriptor("codedeploy", "codedeploy", config.services().codedeploy().enabled(), true,
"codedeploy", storageMode(config.storage().services().codedeploy().mode(), config.storage().mode()),
config.storage().services().codedeploy().flushIntervalMs(), null, ServiceProtocol.JSON,
diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/S3Uri.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/S3Uri.java
new file mode 100644
index 0000000000..941754dd9e
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/S3Uri.java
@@ -0,0 +1,19 @@
+package io.github.hectorvent.floci.services.sagemaker;
+
+record S3Uri(String bucket, String key) {
+ static S3Uri parse(String uri) {
+ if (uri == null || !uri.startsWith("s3://")) {
+ throw new IllegalArgumentException("Expected s3:// URI");
+ }
+ String rest = uri.substring(5);
+ int slash = rest.indexOf('/');
+ String bucket = slash < 0 ? rest : rest.substring(0, slash);
+ if (bucket.isBlank()) {
+ throw new IllegalArgumentException("Expected s3://<bucket>/<key>, got: " + uri);
+ }
+ if (slash < 0) {
+ return new S3Uri(bucket, "");
+ }
+ return new S3Uri(bucket, rest.substring(slash + 1));
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java
new file mode 100644
index 0000000000..ea4c9e2665
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java
@@ -0,0 +1,268 @@
+package io.github.hectorvent.floci.services.sagemaker;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.ContainerTeardown;
+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;
+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.ModelResource;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.jboss.logging.Logger;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.net.ConnectException;
+import java.net.URI;
+import java.net.http.HttpTimeoutException;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@ApplicationScoped
+public class SageMakerEndpointManager implements ContainerTeardown {
+ private static final Logger LOG = Logger.getLogger(SageMakerEndpointManager.class);
+ private static final int PORT = 8080;
+ private static final int DEFAULT_FILE_MODE = 0644;
+ private static final int PING_LOG_FREQUENCY = 15;
+ private static final Duration PING_TIMEOUT = Duration.ofSeconds(120);
+
+ private final ContainerBuilder containerBuilder;
+ private final ContainerLifecycleManager lifecycleManager;
+ private final EmulatorConfig config;
+ private final ContainerDetector containerDetector;
+ private final S3Service s3Service;
+ private final ExecutorService executor = Executors.newCachedThreadPool();
+ private final ConcurrentHashMap<String, String> containers = new ConcurrentHashMap<>();
+ private final HttpClient httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(2)).build();
+
+ @Inject
+ public SageMakerEndpointManager(ContainerBuilder containerBuilder, ContainerLifecycleManager lifecycleManager,
+ EmulatorConfig config, ContainerDetector containerDetector, S3Service s3Service) {
+ this.containerBuilder = containerBuilder;
+ this.lifecycleManager = lifecycleManager;
+ this.config = config;
+ this.containerDetector = containerDetector;
+ this.s3Service = s3Service;
+ }
+
+ public void startEndpointAsync(EndpointResource endpoint, SageMakerService service) {
+ executor.submit(() -> startEndpoint(endpoint, service));
+ }
+
+ private void startEndpoint(EndpointResource endpoint, SageMakerService service) {
+ String containerId = null;
+ try {
+ ModelResource model = service.modelForEndpoint(endpoint);
+ Map<String, Object> c = model.primaryContainer;
+ String image = string(c.get("Image"));
+ if (image == null) {
+ throw new IllegalArgumentException("PrimaryContainer.Image is required");
+ }
+ String name = ContainerStorageHelper.dockerName(config, "floci-sagemaker-endpoint-" + endpoint.endpointName);
+ lifecycleManager.removeIfExists(name);
+ ContainerBuilder.Builder builder = containerBuilder.newContainer(image)
+ .withName(name)
+ .withEnv(environment(endpoint.region, map(c.get("Environment"))))
+ .withDockerNetwork(config.services().sagemaker().dockerNetwork())
+ .withHostDockerInternalOnLinux()
+ .withEmbeddedDns()
+ .withPortBinding(PORT, 0)
+ .withLogRotation();
+ applyEntrypoint(builder, c, "serve");
+ ContainerSpec spec = builder.build();
+ containerId = lifecycleManager.create(spec);
+ String modelDataUrl = string(c.get("ModelDataUrl"));
+ if (modelDataUrl != null && !modelDataUrl.isBlank()) {
+ copyModelData(containerId, modelDataUrl);
+ }
+ ContainerInfo info = lifecycleManager.startCreated(containerId, spec);
+ EndpointInfo ep = info.getEndpoint(PORT);
+ endpoint.containerId = containerId;
+ endpoint.invokeHost = ep.host();
+ endpoint.invokePort = ep.port();
+ containers.put(endpoint.endpointName, containerId);
+ waitForPing(endpoint.invokeHost, endpoint.invokePort);
+ endpoint.endpointStatus = "InService";
+ endpoint.failureReason = null;
+ finishStart(endpoint, service);
+ } catch (Exception e) {
+ LOG.warnv("SageMaker endpoint {0} failed: {1}", endpoint.endpointName, e.getMessage());
+ if (containerId != null) {
+ lifecycleManager.stopAndRemove(containerId, null);
+ }
+ endpoint.endpointStatus = "Failed";
+ endpoint.failureReason = e.getMessage();
+ finishStart(endpoint, service);
+ }
+ }
+
+ /**
+ * Publishes a start attempt's outcome unless a delete or a newer update has superseded it
+ * while Docker was still working: {@link SageMakerService#finalizeEndpointStart} refuses the
+ * write in that case, and the container this attempt just brought up (if any) is torn down
+ * instead of being left running as an orphan nothing points to any more.
+ */
+ private void finishStart(EndpointResource endpoint, SageMakerService service) {
+ if (service.finalizeEndpointStart(endpoint)) {
+ return;
+ }
+ LOG.infov("SageMaker endpoint {0} start superseded by a delete or newer update; discarding result",
+ endpoint.endpointName);
+ String containerId = endpoint.containerId != null ? endpoint.containerId : containers.remove(endpoint.endpointName);
+ if (containerId != null) {
+ lifecycleManager.stopAndRemove(containerId, null);
+ }
+ }
+
+ public void stopEndpoint(EndpointResource endpoint) {
+ String containerId = endpoint.containerId != null ? endpoint.containerId : containers.remove(endpoint.endpointName);
+ if (containerId != null) {
+ lifecycleManager.stopAndRemove(containerId, null);
+ }
+ }
+
+ @Override
+ public void stopManagedContainers() {
+ containers.forEach((name, id) -> lifecycleManager.stopAndRemove(id, null));
+ containers.clear();
+ executor.shutdownNow();
+ }
+
+ private List<String> environment(String region, Map<String, String> modelEnv) {
+ List<String> env = new ArrayList<>();
+ env.add("SAGEMAKER_BIND_TO_PORT=8080");
+ env.add("SAGEMAKER_REGION=" + region);
+ env.add("AWS_REGION=" + region);
+ env.add("AWS_DEFAULT_REGION=" + region);
+ env.add("AWS_ACCESS_KEY_ID=test");
+ env.add("AWS_SECRET_ACCESS_KEY=test");
+ env.add("AWS_SESSION_TOKEN=test");
+ String endpoint = "http://" + resolveEndpointHostname() + ":" + config.port();
+ env.add("AWS_ENDPOINT_URL=" + endpoint);
+ env.add("FLOCI_ENDPOINT=" + endpoint);
+ modelEnv.forEach((k, v) -> env.add(k + "=" + (v == null ? "" : v)));
+ return env;
+ }
+
+ private String resolveEndpointHostname() {
+ return containerDetector.isRunningInContainer() ? config.hostname().orElse(EmbeddedDnsServer.DEFAULT_SUFFIX) : "host.docker.internal";
+ }
+
+ static void applyEntrypoint(ContainerBuilder.Builder builder, Map<String, Object> container, String defaultCommand) {
+ Object entrypoint = container.get("ContainerEntrypoint");
+ Object arguments = container.get("ContainerArguments");
+ if (entrypoint instanceof List<?> list && !list.isEmpty()) {
+ builder.withEntrypoint(list.stream().map(String::valueOf).toList());
+ if (arguments instanceof List<?> args) {
+ builder.withCmd(args.stream().map(String::valueOf).toList());
+ }
+ } else if (arguments instanceof List<?> args && !args.isEmpty()) {
+ builder.withCmd(args.stream().map(String::valueOf).toList());
+ } else {
+ builder.withCmd(List.of(defaultCommand));
+ }
+ }
+
+ private void copyModelData(String containerId, String s3Uri) throws Exception {
+ S3Uri uri = S3Uri.parse(s3Uri);
+ S3Object object = s3Service.getObject(uri.bucket(), uri.key());
+ byte[] tarData = tarModelData(object.getData(), uri.key());
+ lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)
+ .withRemotePath("/")
+ .withTarInputStream(new ByteArrayInputStream(tarData))
+ .exec();
+ }
+
+ private byte[] tarModelData(byte[] data, String key) throws Exception {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (TarArchiveOutputStream tar = new TarArchiveOutputStream(out)) {
+ if (key.endsWith(".tar.gz") || key.endsWith(".tgz")) {
+ try (TarArchiveInputStream in = new TarArchiveInputStream(new GzipCompressorInputStream(new ByteArrayInputStream(data)))) {
+ TarArchiveEntry e;
+ while ((e = in.getNextEntry()) != null) {
+ if (!in.canReadEntryData(e) || e.isDirectory()) continue;
+ String entryName = e.getName();
+ Path normalized = Path.of(entryName).normalize();
+ if (normalized.isAbsolute() || normalized.startsWith("..")) continue;
+ addFile(tar, "opt/ml/model/" + normalized.toString(), in.readAllBytes());
+ }
+ }
+ } else {
+ addFile(tar, "opt/ml/model/" + key.substring(key.lastIndexOf('/') + 1), data);
+ }
+ }
+ return out.toByteArray();
+ }
+
+ static void addFile(TarArchiveOutputStream tar, String path, byte[] data) throws Exception {
+ TarArchiveEntry entry = new TarArchiveEntry(path);
+ entry.setSize(data.length);
+ entry.setMode(DEFAULT_FILE_MODE);
+ tar.putArchiveEntry(entry);
+ tar.write(data);
+ tar.closeArchiveEntry();
+ }
+
+ private void waitForPing(String host, int port) throws Exception {
+ long deadline = System.currentTimeMillis() + PING_TIMEOUT.toMillis();
+ URI uri = URI.create("http://" + host + ":" + port + "/ping");
+ int failures = 0;
+ while (System.currentTimeMillis() < deadline) {
+ try {
+ HttpRequest req = HttpRequest.newBuilder(uri).timeout(Duration.ofSeconds(2)).GET().build();
+ if (httpClient.send(req, HttpResponse.BodyHandlers.discarding()).statusCode() == 200) {
+ return;
+ }
+ failures++;
+ } catch (ConnectException | HttpTimeoutException e) {
+ failures++;
+ if (failures % PING_LOG_FREQUENCY == 0) {
+ LOG.warnv("Still waiting for SageMaker endpoint ping after {0} failures: {1}", failures, e.getMessage());
+ } else {
+ LOG.debugv("Waiting for SageMaker endpoint ping: {0}", e.getMessage());
+ }
+ } catch (Exception e) {
+ failures++;
+ LOG.warnv("Waiting for SageMaker endpoint ping failed: {0}", e.getMessage());
+ }
+ Thread.sleep(1000);
+ }
+ throw new IllegalStateException("Container did not pass /ping health check");
+ }
+
+ @SuppressWarnings("unchecked")
+ static Map<String, String> map(Object value) {
+ if (value instanceof Map<?, ?> in) {
+ LinkedHashMap<String, String> out = new LinkedHashMap<>();
+ in.forEach((k, v) -> out.put(String.valueOf(k), v == null ? "" : String.valueOf(v)));
+ return out;
+ }
+ return Map.of();
+ }
+
+ static String string(Object value) {
+ return value == null ? null : String.valueOf(value);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerJsonHandler.java
new file mode 100644
index 0000000000..87a724a33b
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerJsonHandler.java
@@ -0,0 +1,43 @@
+package io.github.hectorvent.floci.services.sagemaker;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import jakarta.ws.rs.core.Response;
+
+@ApplicationScoped
+public class SageMakerJsonHandler {
+ private final SageMakerService service;
+
+ @Inject
+ public SageMakerJsonHandler(SageMakerService service) {
+ this.service = service;
+ }
+
+ public Response handle(String action, JsonNode request, String region) {
+ Object entity = switch (action) {
+ case "CreateModel" -> service.createModel(request, region);
+ case "DescribeModel" -> service.describeModel(request, region);
+ case "DeleteModel" -> service.deleteModel(request, region);
+ case "ListModels" -> service.listModels(request, region);
+ case "CreateEndpointConfig" -> service.createEndpointConfig(request, region);
+ case "DescribeEndpointConfig" -> service.describeEndpointConfig(request, region);
+ case "DeleteEndpointConfig" -> service.deleteEndpointConfig(request, region);
+ case "ListEndpointConfigs" -> service.listEndpointConfigs(request, region);
+ case "CreateEndpoint" -> service.createEndpoint(request, region);
+ case "DescribeEndpoint" -> service.describeEndpoint(request, region);
+ case "DeleteEndpoint" -> service.deleteEndpoint(request, region);
+ case "ListEndpoints" -> service.listEndpoints(request, region);
+ case "UpdateEndpoint" -> service.updateEndpoint(request, region);
+ case "CreateTrainingJob" -> service.createTrainingJob(request, region);
+ case "DescribeTrainingJob" -> service.describeTrainingJob(request, region);
+ case "ListTrainingJobs" -> service.listTrainingJobs(request, region);
+ case "StopTrainingJob" -> service.stopTrainingJob(request, region);
+ case "AddTags" -> service.addTags(request);
+ case "ListTags" -> service.listTags(request);
+ case "DeleteTags" -> service.deleteTags(request);
+ default -> throw SageMakerService.validation("Action " + action + " is not supported");
+ };
+ return Response.ok(entity).build();
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerRuntimeController.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerRuntimeController.java
new file mode 100644
index 0000000000..97bca1c2f6
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerRuntimeController.java
@@ -0,0 +1,93 @@
+package io.github.hectorvent.floci.services.sagemaker;
+
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;
+import jakarta.inject.Inject;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.HeaderParam;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.HttpHeaders;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+import org.jboss.logging.Logger;
+
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.Map;
+
+@Path("/endpoints")
+public class SageMakerRuntimeController {
+ private static final Logger LOG = Logger.getLogger(SageMakerRuntimeController.class);
+
+ private final SageMakerService service;
+ private final RegionResolver regionResolver;
+ private final HttpClient httpClient;
+
+ @Inject
+ public SageMakerRuntimeController(SageMakerService service, RegionResolver regionResolver) {
+ this.service = service;
+ this.regionResolver = regionResolver;
+ this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build();
+ }
+
+ @POST
+ @Path("/{endpointName}/invocations")
+ @Consumes(MediaType.WILDCARD)
+ @Produces(MediaType.WILDCARD)
+ public Response invoke(@PathParam("endpointName") String endpointName,
+ @HeaderParam("Content-Type") String contentType,
+ @HeaderParam("Accept") String accept,
+ @Context HttpHeaders headers,
+ byte[] body) {
+ String region = regionResolver.resolveRegion(headers);
+ EndpointResource ep = service.endpoint(region, endpointName).orElse(null);
+ if (ep == null) {
+ return validation("Endpoint " + endpointName + " of account " + regionResolver.getAccountId() + " not found.");
+ }
+ if (!"InService".equals(ep.endpointStatus) || ep.invokeHost == null) {
+ return validation("Endpoint " + endpointName + " is not InService.");
+ }
+ try {
+ HttpRequest.Builder builder = HttpRequest.newBuilder(URI.create("http://" + ep.invokeHost + ":" + ep.invokePort + "/invocations"))
+ .timeout(Duration.ofSeconds(60))
+ .POST(HttpRequest.BodyPublishers.ofByteArray(body == null ? new byte[0] : body));
+ if (contentType != null) {
+ builder.header("Content-Type", contentType);
+ }
+ if (accept != null) {
+ builder.header("Accept", accept);
+ }
+ HttpResponse<byte[]> response = httpClient.send(builder.build(), HttpResponse.BodyHandlers.ofByteArray());
+ // SageMaker Runtime maps model-container 5xx responses to ModelError (HTTP 424).
+ if (response.statusCode() >= 500) {
+ String originalMessage = new String(response.body(), StandardCharsets.UTF_8);
+ return Response.status(424).type(MediaType.APPLICATION_JSON).entity(Map.of(
+ "__type", "ModelError",
+ "Message", "Received server error (" + response.statusCode() + ") from primary with message \"" + originalMessage + "\"",
+ "OriginalStatusCode", response.statusCode(),
+ "OriginalMessage", originalMessage
+ )).build();
+ }
+ Response.ResponseBuilder out = Response.status(response.statusCode()).entity(response.body());
+ response.headers().firstValue("Content-Type").ifPresent(out::type);
+ return out.build();
+ } catch (Exception e) {
+ LOG.warnv("SageMaker endpoint invocation failed for {0}: {1}", endpointName, e.getMessage());
+ return Response.status(424).type(MediaType.APPLICATION_JSON)
+ .entity(Map.of("__type", "ModelError", "message", e.getMessage())).build();
+ }
+ }
+
+ private Response validation(String message) {
+ return Response.status(400).type(MediaType.APPLICATION_JSON)
+ .entity(Map.of("__type", "ValidationError", "message", message)).build();
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerService.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerService.java
new file mode 100644
index 0000000000..8962b25c64
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerService.java
@@ -0,0 +1,631 @@
+package io.github.hectorvent.floci.services.sagemaker;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointConfigResource;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.ModelResource;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+
+@ApplicationScoped
+public class SageMakerService {
+ static final int DEFAULT_LIMIT = 100;
+ static final int MAX_RESULTS_CEILING = 100;
+ private static final ObjectMapper STATIC_MAPPER = new ObjectMapper();
+
+ private final StorageBackend<String, ModelResource> modelStore;
+ private final StorageBackend<String, EndpointConfigResource> endpointConfigStore;
+ private final StorageBackend<String, EndpointResource> endpointStore;
+ private final StorageBackend<String, TrainingJobResource> trainingJobStore;
+ private final RegionResolver regionResolver;
+ private final ObjectMapper mapper;
+ private final SageMakerEndpointManager endpointManager;
+ private final SageMakerTrainingRunner trainingRunner;
+ private final Clock clock;
+
+ @Inject
+ public SageMakerService(StorageFactory storageFactory, RegionResolver regionResolver, ObjectMapper mapper,
+ SageMakerEndpointManager endpointManager, SageMakerTrainingRunner trainingRunner) {
+ this(storageFactory.create("sagemaker", "sagemaker-models.json", new TypeReference<Map<String, ModelResource>>() {}),
+ storageFactory.create("sagemaker", "sagemaker-endpoint-configs.json", new TypeReference<Map<String, EndpointConfigResource>>() {}),
+ storageFactory.create("sagemaker", "sagemaker-endpoints.json", new TypeReference<Map<String, EndpointResource>>() {}),
+ storageFactory.create("sagemaker", "sagemaker-training-jobs.json", new TypeReference<Map<String, TrainingJobResource>>() {}),
+ regionResolver, mapper, endpointManager, trainingRunner, Clock.systemUTC());
+ }
+
+ SageMakerService(StorageBackend<String, ModelResource> modelStore,
+ StorageBackend<String, EndpointConfigResource> endpointConfigStore,
+ StorageBackend<String, EndpointResource> endpointStore,
+ StorageBackend<String, TrainingJobResource> trainingJobStore,
+ RegionResolver regionResolver,
+ ObjectMapper mapper,
+ SageMakerEndpointManager endpointManager,
+ SageMakerTrainingRunner trainingRunner) {
+ this(modelStore, endpointConfigStore, endpointStore, trainingJobStore, regionResolver, mapper, endpointManager,
+ trainingRunner, Clock.systemUTC());
+ }
+
+ SageMakerService(StorageBackend<String, ModelResource> modelStore,
+ StorageBackend<String, EndpointConfigResource> endpointConfigStore,
+ StorageBackend<String, EndpointResource> endpointStore,
+ StorageBackend<String, TrainingJobResource> trainingJobStore,
+ RegionResolver regionResolver,
+ ObjectMapper mapper,
+ SageMakerEndpointManager endpointManager,
+ SageMakerTrainingRunner trainingRunner,
+ Clock clock) {
+ this.modelStore = modelStore;
+ this.endpointConfigStore = endpointConfigStore;
+ this.endpointStore = endpointStore;
+ this.trainingJobStore = trainingJobStore;
+ this.regionResolver = regionResolver;
+ this.mapper = mapper;
+ this.endpointManager = endpointManager;
+ this.trainingRunner = trainingRunner;
+ this.clock = clock;
+ }
+
+ public synchronized ObjectNode createModel(JsonNode request, String region) {
+ String name = required(request, "ModelName");
+ if (model(region, name).isPresent()) {
+ throw new AwsException("ValidationException", "Cannot create already existing model \"" + name + "\".", 400);
+ }
+ JsonNode primary = request.path("PrimaryContainer");
+ JsonNode containers = request.path("Containers");
+ boolean hasContainers = containers.isArray() && !containers.isEmpty() && containers.get(0).isObject();
+ if (!primary.isObject() && !hasContainers) {
+ throw validation("PrimaryContainer or Containers is required");
+ }
+ ModelResource model = new ModelResource();
+ model.modelName = name;
+ model.modelArn = arn(region, "model/" + name);
+ model.executionRoleArn = text(request, "ExecutionRoleArn");
+ model.creationTime = nowMillis();
+ model.region = region;
+ model.accountId = regionResolver.getAccountId();
+ model.primaryContainer = primary.isObject() ? map(primary) : map(containers.get(0));
+ model.containers = listMap(containers);
+ model.tags = tagsFromList(request.path("Tags"));
+ modelStore.put(regionKey(region, name), model);
+ ObjectNode out = mapper.createObjectNode();
+ out.put("ModelArn", model.modelArn);
+ return out;
+ }
+
+ public ObjectNode describeModel(JsonNode request, String region) {
+ String name = required(request, "ModelName");
+ ModelResource m = model(region, name).orElseThrow(() -> validation("Could not find model \"" + name + "\"."));
+ ObjectNode out = mapper.createObjectNode();
+ out.put("ModelName", m.modelName);
+ out.put("ModelArn", m.modelArn);
+ out.set("PrimaryContainer", mapper.valueToTree(m.primaryContainer));
+ if (!m.containers.isEmpty()) {
+ out.set("Containers", mapper.valueToTree(m.containers));
+ }
+ if (m.executionRoleArn != null) {
+ out.put("ExecutionRoleArn", m.executionRoleArn);
+ }
+ out.put("CreationTime", epoch(m.creationTime));
+ return out;
+ }
+
+ public synchronized ObjectNode deleteModel(JsonNode request, String region) {
+ String name = required(request, "ModelName");
+ model(region, name).orElseThrow(() -> validation("Could not find model \"" + name + "\"."));
+ modelStore.delete(regionKey(region, name));
+ return mapper.createObjectNode();
+ }
+
+ public ObjectNode listModels(JsonNode request, String region) {
+ List<ModelResource> models = modelStore.scan(k -> k.startsWith(region + "::")).stream()
+ .sorted(Comparator.comparing(m -> m.modelName)).toList();
+ String nameContains = text(request, "NameContains");
+ if (nameContains != null && !nameContains.isBlank()) {
+ models = models.stream().filter(m -> m.modelName.contains(nameContains)).toList();
+ }
+ return paginate(models, request, m -> m.modelName, (m, n) -> {
+ n.put("ModelName", m.modelName);
+ n.put("ModelArn", m.modelArn);
+ n.put("CreationTime", epoch(m.creationTime));
+ }, "Models");
+ }
+
+ public synchronized ObjectNode createEndpointConfig(JsonNode request, String region) {
+ String name = required(request, "EndpointConfigName");
+ if (endpointConfig(region, name).isPresent()) {
+ throw new AwsException("ValidationException", "Cannot create already existing endpoint configuration \"" + name + "\".", 400);
+ }
+ List<Map<String, Object>> variants = listMap(request.path("ProductionVariants"));
+ if (variants.isEmpty()) {
+ throw validation("ProductionVariants is required");
+ }
+ for (Map<String, Object> v : variants) {
+ String modelName = SageMakerEndpointManager.string(v.get("ModelName"));
+ if (modelName == null || modelName.isBlank()) {
+ throw validation("ProductionVariants.ModelName is required");
+ }
+ if (model(region, modelName).isEmpty()) {
+ throw validation("Could not find model \"" + modelName + "\".");
+ }
+ }
+ EndpointConfigResource cfg = new EndpointConfigResource();
+ cfg.endpointConfigName = name;
+ cfg.endpointConfigArn = arn(region, "endpoint-config/" + name);
+ cfg.productionVariants = variants;
+ cfg.creationTime = nowMillis();
+ cfg.region = region;
+ cfg.accountId = regionResolver.getAccountId();
+ cfg.tags = tagsFromList(request.path("Tags"));
+ endpointConfigStore.put(regionKey(region, name), cfg);
+ ObjectNode out = mapper.createObjectNode();
+ out.put("EndpointConfigArn", cfg.endpointConfigArn);
+ return out;
+ }
+
+ public ObjectNode describeEndpointConfig(JsonNode request, String region) {
+ String name = required(request, "EndpointConfigName");
+ EndpointConfigResource cfg = endpointConfig(region, name)
+ .orElseThrow(() -> validation("Could not find endpoint configuration \"" + name + "\"."));
+ ObjectNode out = mapper.createObjectNode();
+ out.put("EndpointConfigName", cfg.endpointConfigName);
+ out.put("EndpointConfigArn", cfg.endpointConfigArn);
+ out.set("ProductionVariants", mapper.valueToTree(cfg.productionVariants));
+ out.put("CreationTime", epoch(cfg.creationTime));
+ return out;
+ }
+
+ public synchronized ObjectNode deleteEndpointConfig(JsonNode request, String region) {
+ String name = required(request, "EndpointConfigName");
+ endpointConfig(region, name).orElseThrow(() -> validation("Could not find endpoint configuration \"" + name + "\"."));
+ endpointConfigStore.delete(regionKey(region, name));
+ return mapper.createObjectNode();
+ }
+
+ public ObjectNode listEndpointConfigs(JsonNode request, String region) {
+ List<EndpointConfigResource> configs = endpointConfigStore.scan(k -> k.startsWith(region + "::")).stream()
+ .sorted(Comparator.comparing(c -> c.endpointConfigName)).toList();
+ return paginate(configs, request, c -> c.endpointConfigName, (c, n) -> {
+ n.put("EndpointConfigName", c.endpointConfigName);
+ n.put("EndpointConfigArn", c.endpointConfigArn);
+ n.put("CreationTime", epoch(c.creationTime));
+ }, "EndpointConfigs");
+ }
+
+ public synchronized ObjectNode createEndpoint(JsonNode request, String region) {
+ String name = required(request, "EndpointName");
+ if (endpoint(region, name).isPresent()) {
+ throw new AwsException("ValidationException", "Cannot create already existing endpoint \"" + name + "\".", 400);
+ }
+ String cfgName = required(request, "EndpointConfigName");
+ endpointConfig(region, cfgName).orElseThrow(() -> validation("Could not find endpoint configuration \"" + cfgName + "\"."));
+ EndpointResource ep = new EndpointResource();
+ ep.endpointName = name;
+ ep.endpointArn = arn(region, "endpoint/" + name);
+ ep.endpointConfigName = cfgName;
+ ep.endpointStatus = "Creating";
+ ep.creationTime = nowMillis();
+ ep.lastModifiedTime = ep.creationTime;
+ ep.region = region;
+ ep.accountId = regionResolver.getAccountId();
+ ep.tags = tagsFromList(request.path("Tags"));
+ ep.generation = 1;
+ endpointStore.put(regionKey(region, name), ep);
+ endpointManager.startEndpointAsync(ep, this);
+ ObjectNode out = mapper.createObjectNode();
+ out.put("EndpointArn", ep.endpointArn);
+ return out;
+ }
+
+ public synchronized ObjectNode updateEndpoint(JsonNode request, String region) {
+ String name = required(request, "EndpointName");
+ EndpointResource ep = endpoint(region, name).orElseThrow(() -> validation("Could not find endpoint \"" + name + "\"."));
+ String cfgName = required(request, "EndpointConfigName");
+ endpointConfig(region, cfgName).orElseThrow(() -> validation("Could not find endpoint configuration \"" + cfgName + "\"."));
+ if (ep.containerId != null) {
+ endpointManager.stopEndpoint(ep);
+ }
+ ep.endpointConfigName = cfgName;
+ ep.endpointStatus = "Creating";
+ ep.failureReason = null;
+ ep.containerId = null;
+ ep.lastModifiedTime = nowMillis();
+ ep.generation++;
+ endpointStore.put(regionKey(region, name), ep);
+ endpointManager.startEndpointAsync(ep, this);
+ ObjectNode out = mapper.createObjectNode();
+ out.put("EndpointArn", ep.endpointArn);
+ return out;
+ }
+
+ public ObjectNode describeEndpoint(JsonNode request, String region) {
+ String name = required(request, "EndpointName");
+ EndpointResource ep = endpoint(region, name).orElseThrow(() -> validation("Could not find endpoint \"" + name + "\"."));
+ ObjectNode out = mapper.createObjectNode();
+ out.put("EndpointName", ep.endpointName);
+ out.put("EndpointArn", ep.endpointArn);
+ out.put("EndpointConfigName", ep.endpointConfigName);
+ out.put("EndpointStatus", ep.endpointStatus);
+ if (ep.failureReason != null) {
+ out.put("FailureReason", ep.failureReason);
+ }
+ out.put("CreationTime", epoch(ep.creationTime));
+ out.put("LastModifiedTime", epoch(ep.lastModifiedTime));
+ return out;
+ }
+
+ public synchronized ObjectNode deleteEndpoint(JsonNode request, String region) {
+ String name = required(request, "EndpointName");
+ EndpointResource ep = endpoint(region, name).orElseThrow(() -> validation("Could not find endpoint \"" + name + "\"."));
+ ep.endpointStatus = "Deleting";
+ // Bump the generation before removing the record: a start worker still in flight for
+ // this endpoint checks the generation against what is (no longer) in the store and will
+ // no-op instead of resurrecting the record it raced with this delete.
+ ep.generation++;
+ endpointStore.put(regionKey(region, name), ep);
+ endpointManager.stopEndpoint(ep);
+ endpointStore.delete(regionKey(region, name));
+ return mapper.createObjectNode();
+ }
+
+ public ObjectNode listEndpoints(JsonNode request, String region) {
+ List<EndpointResource> endpoints = endpointStore.scan(k -> k.startsWith(region + "::")).stream()
+ .sorted(Comparator.comparing(e -> e.endpointName)).toList();
+ return paginate(endpoints, request, e -> e.endpointName, (e, n) -> {
+ n.put("EndpointName", e.endpointName);
+ n.put("EndpointArn", e.endpointArn);
+ n.put("EndpointStatus", e.endpointStatus);
+ n.put("CreationTime", epoch(e.creationTime));
+ n.put("LastModifiedTime", epoch(e.lastModifiedTime));
+ }, "Endpoints");
+ }
+
+ public synchronized ObjectNode createTrainingJob(JsonNode request, String region) {
+ String name = required(request, "TrainingJobName");
+ if (trainingJob(region, name).isPresent()) {
+ throw new AwsException("ResourceInUse", "Training job already exists: " + name, 400);
+ }
+ if (!request.path("AlgorithmSpecification").isObject() || text(request.path("AlgorithmSpecification"), "TrainingImage") == null) {
+ throw validation("AlgorithmSpecification.TrainingImage is required");
+ }
+ if (!request.path("OutputDataConfig").isObject() || text(request.path("OutputDataConfig"), "S3OutputPath") == null) {
+ throw validation("OutputDataConfig.S3OutputPath is required");
+ }
+ validateS3Uri(text(request.path("OutputDataConfig"), "S3OutputPath"), "OutputDataConfig.S3OutputPath");
+ for (JsonNode channel : request.path("InputDataConfig")) {
+ JsonNode s3DataSource = channel.path("DataSource").path("S3DataSource");
+ if (s3DataSource.isObject()) {
+ validateS3Uri(text(s3DataSource, "S3Uri"), "InputDataConfig[].DataSource.S3DataSource.S3Uri");
+ }
+ }
+ TrainingJobResource job = new TrainingJobResource();
+ job.trainingJobName = name;
+ job.trainingJobArn = arn(region, "training-job/" + name);
+ job.trainingJobStatus = "InProgress";
+ job.secondaryStatus = "Starting";
+ job.creationTime = nowMillis();
+ job.region = region;
+ job.accountId = regionResolver.getAccountId();
+ job.algorithmSpecification = map(request.path("AlgorithmSpecification"));
+ job.inputDataConfig = listMap(request.path("InputDataConfig"));
+ job.outputDataConfig = map(request.path("OutputDataConfig"));
+ job.resourceConfig = map(request.path("ResourceConfig"));
+ job.stoppingCondition = map(request.path("StoppingCondition"));
+ job.hyperParameters = stringMap(request.path("HyperParameters"));
+ job.tags = tagsFromList(request.path("Tags"));
+ trainingJobStore.put(regionKey(region, name), job);
+ trainingRunner.runAsync(job, this);
+ ObjectNode out = mapper.createObjectNode();
+ out.put("TrainingJobArn", job.trainingJobArn);
+ return out;
+ }
+
+ public ObjectNode describeTrainingJob(JsonNode request, String region) {
+ String name = required(request, "TrainingJobName");
+ TrainingJobResource job = trainingJob(region, name).orElseThrow(() -> validation("Could not find training job \"" + name + "\"."));
+ ObjectNode out = mapper.valueToTree(Map.of(
+ "TrainingJobName", job.trainingJobName,
+ "TrainingJobArn", job.trainingJobArn,
+ "TrainingJobStatus", job.trainingJobStatus,
+ "SecondaryStatus", job.secondaryStatus == null ? "" : job.secondaryStatus,
+ "AlgorithmSpecification", job.algorithmSpecification,
+ "InputDataConfig", job.inputDataConfig,
+ "OutputDataConfig", job.outputDataConfig,
+ "ResourceConfig", job.resourceConfig,
+ "StoppingCondition", job.stoppingCondition,
+ "CreationTime", epoch(job.creationTime)
+ ));
+ if (job.trainingStartTime > 0) {
+ out.put("TrainingStartTime", epoch(job.trainingStartTime));
+ }
+ if (job.trainingEndTime > 0) {
+ out.put("TrainingEndTime", epoch(job.trainingEndTime));
+ }
+ if (job.failureReason != null) {
+ out.put("FailureReason", job.failureReason);
+ }
+ if (job.modelArtifactsS3ModelArtifacts != null) {
+ ObjectNode ma = mapper.createObjectNode();
+ ma.put("S3ModelArtifacts", job.modelArtifactsS3ModelArtifacts);
+ out.set("ModelArtifacts", ma);
+ }
+ return out;
+ }
+
+ public ObjectNode listTrainingJobs(JsonNode request, String region) {
+ List<TrainingJobResource> jobs = trainingJobStore.scan(k -> k.startsWith(region + "::")).stream()
+ .sorted(Comparator.comparing(j -> j.trainingJobName)).toList();
+ String nameContains = text(request, "NameContains");
+ if (nameContains != null && !nameContains.isBlank()) {
+ jobs = jobs.stream().filter(j -> j.trainingJobName.contains(nameContains)).toList();
+ }
+ String statusEquals = text(request, "StatusEquals");
+ if (statusEquals != null && !statusEquals.isBlank()) {
+ jobs = jobs.stream().filter(j -> statusEquals.equals(j.trainingJobStatus)).toList();
+ }
+ return paginate(jobs, request, j -> j.trainingJobName, (j, n) -> {
+ n.put("TrainingJobName", j.trainingJobName);
+ n.put("TrainingJobArn", j.trainingJobArn);
+ n.put("TrainingJobStatus", j.trainingJobStatus);
+ n.put("CreationTime", epoch(j.creationTime));
+ }, "TrainingJobSummaries");
+ }
+
+ public synchronized ObjectNode stopTrainingJob(JsonNode request, String region) {
+ String name = required(request, "TrainingJobName");
+ TrainingJobResource job = trainingJob(region, name).orElseThrow(() -> validation("Could not find training job \"" + name + "\"."));
+ // AWS only transitions InProgress/Stopping jobs; a job already in a terminal state is
+ // left as-is rather than relabeled "Stopped" out from under its real outcome.
+ if ("InProgress".equals(job.trainingJobStatus) || "Stopping".equals(job.trainingJobStatus)) {
+ trainingRunner.stop(job);
+ job.trainingJobStatus = "Stopped";
+ job.secondaryStatus = "Stopped";
+ job.trainingEndTime = nowMillis();
+ trainingJobStore.put(regionKey(region, name), job);
+ }
+ return mapper.createObjectNode();
+ }
+
+ public synchronized ObjectNode addTags(JsonNode request) {
+ String arn = required(request, "ResourceArn");
+ Map<String, String> tags = tagsFromList(request.path("Tags"));
+ resourceTags(arn).putAll(tags);
+ persistByArn(arn);
+ ObjectNode out = mapper.createObjectNode();
+ out.set("Tags", tagsArray(resourceTags(arn)));
+ return out;
+ }
+
+ public ObjectNode listTags(JsonNode request) {
+ String arn = required(request, "ResourceArn");
+ ObjectNode out = mapper.createObjectNode();
+ out.set("Tags", tagsArray(resourceTags(arn)));
+ return out;
+ }
+
+ public synchronized ObjectNode deleteTags(JsonNode request) {
+ String arn = required(request, "ResourceArn");
+ Map<String, String> tags = resourceTags(arn);
+ request.path("TagKeys").forEach(k -> tags.remove(k.asText()));
+ persistByArn(arn);
+ return mapper.createObjectNode();
+ }
+
+ Optional<ModelResource> model(String region, String name) {
+ return modelStore.get(regionKey(region, name));
+ }
+
+ Optional<EndpointConfigResource> endpointConfig(String region, String name) {
+ return endpointConfigStore.get(regionKey(region, name));
+ }
+
+ public Optional<EndpointResource> endpoint(String region, String name) {
+ return endpointStore.get(regionKey(region, name));
+ }
+
+ Optional<TrainingJobResource> trainingJob(String region, String name) {
+ return trainingJobStore.get(regionKey(region, name));
+ }
+
+ public ModelResource modelForEndpoint(EndpointResource ep) {
+ EndpointConfigResource cfg = endpointConfig(ep.region, ep.endpointConfigName)
+ .orElseThrow(() -> validation("Could not find endpoint configuration \"" + ep.endpointConfigName + "\"."));
+ String modelName = String.valueOf(cfg.productionVariants.get(0).get("ModelName"));
+ return model(ep.region, modelName).orElseThrow(() -> validation("Could not find model \"" + modelName + "\"."));
+ }
+
+ /**
+ * Persists the result of an endpoint start (success or failure) unless a delete or a
+ * newer start has superseded it since {@code ep} was handed to {@link SageMakerEndpointManager}.
+ * Returns {@code false} when the caller should tear down whatever it just started instead
+ * of publishing it, because the endpoint it belongs to is gone or has moved on.
+ */
+ synchronized boolean finalizeEndpointStart(EndpointResource ep) {
+ EndpointResource current = endpointStore.get(regionKey(ep.region, ep.endpointName)).orElse(null);
+ if (current == null || current.generation != ep.generation) {
+ return false;
+ }
+ ep.lastModifiedTime = nowMillis();
+ endpointStore.put(regionKey(ep.region, ep.endpointName), ep);
+ return true;
+ }
+
+ public synchronized void updateTrainingJob(TrainingJobResource job) {
+ trainingJobStore.put(regionKey(job.region, job.trainingJobName), job);
+ }
+
+ private Map<String, String> resourceTags(String arn) {
+ String region = arnRegion(arn);
+ String name = arn.substring(arn.lastIndexOf('/') + 1);
+ if (arn.contains(":model/")) return model(region, name).orElseThrow(() -> validation("Could not find model \"" + name + "\".")).tags;
+ if (arn.contains(":endpoint-config/")) return endpointConfig(region, name).orElseThrow(() -> validation("Could not find endpoint configuration \"" + name + "\".")).tags;
+ if (arn.contains(":endpoint/")) return endpoint(region, name).orElseThrow(() -> validation("Could not find endpoint \"" + name + "\".")).tags;
+ if (arn.contains(":training-job/")) return trainingJob(region, name).orElseThrow(() -> validation("Could not find training job \"" + name + "\".")).tags;
+ throw validation("ResourceArn is invalid");
+ }
+
+ private void persistByArn(String arn) {
+ String region = arnRegion(arn);
+ String name = arn.substring(arn.lastIndexOf('/') + 1);
+ if (arn.contains(":model/")) model(region, name).ifPresent(v -> modelStore.put(regionKey(region, name), v));
+ else if (arn.contains(":endpoint-config/")) endpointConfig(region, name).ifPresent(v -> endpointConfigStore.put(regionKey(region, name), v));
+ else if (arn.contains(":endpoint/")) endpoint(region, name).ifPresent(v -> endpointStore.put(regionKey(region, name), v));
+ else if (arn.contains(":training-job/")) trainingJob(region, name).ifPresent(v -> trainingJobStore.put(regionKey(region, name), v));
+ }
+
+ /** ARN shape: {@code arn:aws:sagemaker:<region>:<account>:<type>/<name>}. */
+ private static String arnRegion(String arn) {
+ String[] parts = arn.split(":", 6);
+ if (parts.length < 4 || parts[3].isBlank()) {
+ throw validation("ResourceArn is invalid");
+ }
+ return parts[3];
+ }
+
+ private static String regionKey(String region, String name) {
+ return region + "::" + name;
+ }
+
+ private String arn(String region, String resource) {
+ return "arn:aws:sagemaker:" + region + ":" + regionResolver.getAccountId() + ":" + resource;
+ }
+
+ private long nowMillis() {
+ return clock.millis();
+ }
+
+ static double epoch(long millis) {
+ return millis / 1000.0;
+ }
+
+ static AwsException validation(String message) {
+ return new AwsException("ValidationException", message, 400);
+ }
+
+ static void validateS3Uri(String uri, String field) {
+ try {
+ S3Uri.parse(uri);
+ } catch (IllegalArgumentException e) {
+ throw validation(field + ": " + e.getMessage());
+ }
+ }
+
+ static String required(JsonNode node, String field) {
+ String value = text(node, field);
+ if (value == null || value.isBlank()) {
+ throw validation(field + " is required");
+ }
+ return value;
+ }
+
+ static String text(JsonNode node, String field) {
+ JsonNode v = node.path(field);
+ return v.isMissingNode() || v.isNull() ? null : v.asText();
+ }
+
+ static Map<String, Object> map(JsonNode node) {
+ if (!node.isObject()) {
+ return new LinkedHashMap<>();
+ }
+ return STATIC_MAPPER.convertValue(node, new TypeReference<LinkedHashMap<String, Object>>() {});
+ }
+
+ static Map<String, String> stringMap(JsonNode node) {
+ Map<String, String> out = new LinkedHashMap<>();
+ if (node.isObject()) {
+ node.fields().forEachRemaining(e -> out.put(e.getKey(), e.getValue().asText()));
+ }
+ return out;
+ }
+
+ static List<Map<String, Object>> listMap(JsonNode node) {
+ List<Map<String, Object>> out = new ArrayList<>();
+ if (node.isArray()) {
+ node.forEach(n -> out.add(map(n)));
+ }
+ return out;
+ }
+
+ static Map<String, String> tagsFromList(JsonNode node) {
+ Map<String, String> out = new LinkedHashMap<>();
+ if (node.isArray()) {
+ node.forEach(t -> {
+ String key = text(t, "Key");
+ if (key != null && !key.isBlank()) {
+ out.put(key, text(t, "Value"));
+ }
+ });
+ }
+ return out;
+ }
+
+ private ArrayNode tagsArray(Map<String, String> tags) {
+ ArrayNode arr = mapper.createArrayNode();
+ tags.forEach((k, v) -> {
+ ObjectNode n = mapper.createObjectNode();
+ n.put("Key", k);
+ n.put("Value", v);
+ arr.add(n);
+ });
+ return arr;
+ }
+
+ /**
+ * Applies AWS's {@code MaxResults}/{@code NextToken} contract to an already-filtered,
+ * name-sorted list: {@code MaxResults} (1-100, default {@value #DEFAULT_LIMIT}) caps the
+ * page, and an opaque {@code NextToken} (the last name returned) resumes strictly after it,
+ * matching the sort the caller applied.
+ */
+ private <T> ObjectNode paginate(List<T> sorted, JsonNode request, Function<T, String> nameOf,
+ BiConsumer<T, ObjectNode> render, String itemsField) {
+ int max = maxResults(request);
+ String nextToken = text(request, "NextToken");
+ List<T> remaining = sorted;
+ if (nextToken != null && !nextToken.isBlank()) {
+ remaining = remaining.stream().dropWhile(v -> nameOf.apply(v).compareTo(nextToken) <= 0).toList();
+ }
+ List<T> page = remaining.stream().limit(max).toList();
+ ArrayNode arr = mapper.createArrayNode();
+ page.forEach(v -> {
+ ObjectNode n = mapper.createObjectNode();
+ render.accept(v, n);
+ arr.add(n);
+ });
+ ObjectNode out = mapper.createObjectNode();
+ out.set(itemsField, arr);
+ if (page.size() == max && remaining.size() > max) {
+ out.put("NextToken", nameOf.apply(page.get(page.size() - 1)));
+ }
+ return out;
+ }
+
+ private static int maxResults(JsonNode request) {
+ JsonNode node = request.path("MaxResults");
+ if (node.isMissingNode() || node.isNull()) {
+ return DEFAULT_LIMIT;
+ }
+ int max = node.asInt();
+ if (max < 1 || max > MAX_RESULTS_CEILING) {
+ throw validation("MaxResults must be between 1 and " + MAX_RESULTS_CEILING + ".");
+ }
+ return max;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java
new file mode 100644
index 0000000000..de6a063964
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java
@@ -0,0 +1,362 @@
+package io.github.hectorvent.floci.services.sagemaker;
+
+import com.github.dockerjava.api.command.InspectContainerResponse;
+import com.github.dockerjava.api.exception.NotFoundException;
+import com.github.dockerjava.api.model.Frame;
+import com.github.dockerjava.core.command.LogContainerResultCallback;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.ContainerTeardown;
+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+import org.jboss.logging.Logger;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@ApplicationScoped
+public class SageMakerTrainingRunner implements ContainerTeardown {
+ private static final Logger LOG = Logger.getLogger(SageMakerTrainingRunner.class);
+ private static final String LOG_GROUP = "/aws/sagemaker/TrainingJobs";
+
+ private final ContainerBuilder containerBuilder;
+ private final ContainerLifecycleManager lifecycleManager;
+ private final ContainerLogStreamer logStreamer;
+ private final EmulatorConfig config;
+ private final ContainerDetector containerDetector;
+ private final S3Service s3Service;
+ private final ObjectMapper mapper;
+ private final ExecutorService executor = Executors.newCachedThreadPool();
+ private final ConcurrentHashMap<String, String> containers = new ConcurrentHashMap<>();
+ // Names a stop that stop() has already committed to the store as "Stopped": run()'s exit-code
+ // handling checks this before persisting, since a stop races the removal of the very container
+ // waitForExit() is polling, and the container disappearing (NotFoundException, exit code 1)
+ // must not be relabeled a training failure.
+ private final Set<String> stopRequested = ConcurrentHashMap.newKeySet();
+
+ @Inject
+ public SageMakerTrainingRunner(ContainerBuilder containerBuilder, ContainerLifecycleManager lifecycleManager,
+ ContainerLogStreamer logStreamer, EmulatorConfig config,
+ ContainerDetector containerDetector, S3Service s3Service, ObjectMapper mapper) {
+ this.containerBuilder = containerBuilder;
+ this.lifecycleManager = lifecycleManager;
+ this.logStreamer = logStreamer;
+ this.config = config;
+ this.containerDetector = containerDetector;
+ this.s3Service = s3Service;
+ this.mapper = mapper;
+ }
+
+ public void runAsync(TrainingJobResource job, SageMakerService service) {
+ executor.submit(() -> run(job, service));
+ }
+
+ private void run(TrainingJobResource job, SageMakerService service) {
+ String containerId = null;
+ Closeable logs = null;
+ try {
+ String image = SageMakerEndpointManager.string(job.algorithmSpecification.get("TrainingImage"));
+ String name = ContainerStorageHelper.dockerName(config, "floci-sagemaker-training-" + job.trainingJobName);
+ lifecycleManager.removeIfExists(name);
+ ContainerBuilder.Builder builder = containerBuilder.newContainer(image)
+ .withName(name)
+ .withEnv(environment(job))
+ .withDockerNetwork(config.services().sagemaker().dockerNetwork())
+ .withHostDockerInternalOnLinux()
+ .withEmbeddedDns()
+ .withLogRotation();
+ SageMakerEndpointManager.applyEntrypoint(builder, job.algorithmSpecification, "train");
+ ContainerSpec spec = builder.build();
+ containerId = lifecycleManager.create(spec);
+ containers.put(job.trainingJobName, containerId);
+ job.containerId = containerId;
+ copyTrainingInput(containerId, job);
+ lifecycleManager.startCreated(containerId, spec);
+ job.trainingStartTime = System.currentTimeMillis();
+ job.secondaryStatus = "Training";
+ service.updateTrainingJob(job);
+ logs = logStreamer.attach(containerId, LOG_GROUP, job.trainingJobName + "/algo-1", job.region,
+ "sagemaker:" + job.trainingJobName);
+ Integer exit = waitForExit(containerId, timeout(job));
+ if (stopRequested.remove(job.trainingJobName)) {
+ // stop() already persisted Stopped and is tearing the container down itself;
+ // whatever exit code we just observed (or failed to) is a side effect of that,
+ // not a real outcome to report.
+ containers.remove(job.trainingJobName);
+ return;
+ }
+ if (exit == null) {
+ lifecycleManager.stopAndRemove(containerId, logs);
+ job.trainingJobStatus = "Failed";
+ job.secondaryStatus = "Failed";
+ job.failureReason = "Training job exceeded StoppingCondition.MaxRuntimeInSeconds";
+ } else if (exit == 0) {
+ job.modelArtifactsS3ModelArtifacts = uploadModelArtifacts(containerId, job);
+ lifecycleManager.stopAndRemove(containerId, logs);
+ job.trainingJobStatus = "Completed";
+ job.secondaryStatus = "Completed";
+ } else {
+ job.failureReason = readFailure(containerId, exit);
+ lifecycleManager.stopAndRemove(containerId, logs);
+ job.trainingJobStatus = "Failed";
+ job.secondaryStatus = "Failed";
+ }
+ containers.remove(job.trainingJobName);
+ job.trainingEndTime = System.currentTimeMillis();
+ service.updateTrainingJob(job);
+ } catch (Exception e) {
+ if (stopRequested.remove(job.trainingJobName)) {
+ containers.remove(job.trainingJobName);
+ return;
+ }
+ LOG.warnv("SageMaker training job {0} failed: {1}", job.trainingJobName, e.getMessage());
+ if (containerId != null) {
+ lifecycleManager.stopAndRemove(containerId, logs);
+ }
+ containers.remove(job.trainingJobName);
+ job.trainingJobStatus = "Failed";
+ job.secondaryStatus = "Failed";
+ job.failureReason = e.getMessage();
+ job.trainingEndTime = System.currentTimeMillis();
+ service.updateTrainingJob(job);
+ }
+ }
+
+ public void stop(TrainingJobResource job) {
+ stopRequested.add(job.trainingJobName);
+ String id = job.containerId != null ? job.containerId : containers.remove(job.trainingJobName);
+ if (id != null) {
+ lifecycleManager.stopAndRemove(id, null);
+ }
+ }
+
+ @Override
+ public void stopManagedContainers() {
+ containers.forEach((name, id) -> lifecycleManager.stopAndRemove(id, null));
+ containers.clear();
+ executor.shutdownNow();
+ }
+
+ private List<String> environment(TrainingJobResource job) {
+ List<String> env = new ArrayList<>();
+ env.add("TRAINING_JOB_NAME=" + job.trainingJobName);
+ env.add("TRAINING_JOB_ARN=" + job.trainingJobArn);
+ env.add("SAGEMAKER_REGION=" + job.region);
+ env.add("AWS_REGION=" + job.region);
+ env.add("AWS_DEFAULT_REGION=" + job.region);
+ env.add("AWS_ACCESS_KEY_ID=test");
+ env.add("AWS_SECRET_ACCESS_KEY=test");
+ env.add("AWS_SESSION_TOKEN=test");
+ String endpoint = "http://" + resolveEndpointHostname() + ":" + config.port();
+ env.add("AWS_ENDPOINT_URL=" + endpoint);
+ env.add("FLOCI_ENDPOINT=" + endpoint);
+ return env;
+ }
+
+ private String resolveEndpointHostname() {
+ return containerDetector.isRunningInContainer() ? config.hostname().orElse(EmbeddedDnsServer.DEFAULT_SUFFIX) : "host.docker.internal";
+ }
+
+ private void copyTrainingInput(String containerId, TrainingJobResource job) throws Exception {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (TarArchiveOutputStream tar = new TarArchiveOutputStream(out)) {
+ SageMakerEndpointManager.addFile(tar, "opt/ml/input/config/hyperparameters.json",
+ mapper.writeValueAsBytes(job.hyperParameters));
+ SageMakerEndpointManager.addFile(tar, "opt/ml/input/config/resourceconfig.json",
+ "{\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\"]}".getBytes(StandardCharsets.UTF_8));
+ SageMakerEndpointManager.addFile(tar, "opt/ml/input/config/inputdataconfig.json",
+ mapper.writeValueAsBytes(inputDataConfig(job.inputDataConfig)));
+ for (Map<String, Object> channel : job.inputDataConfig) {
+ copyChannelObjects(tar, channel);
+ }
+ }
+ lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)
+ .withRemotePath("/")
+ .withTarInputStream(new ByteArrayInputStream(out.toByteArray()))
+ .exec();
+ }
+
+ private Map<String, Object> inputDataConfig(List<Map<String, Object>> channels) {
+ Map<String, Object> out = new LinkedHashMap<>();
+ for (Map<String, Object> c : channels) {
+ Map<String, Object> value = new LinkedHashMap<>();
+ value.put("ContentType", c.getOrDefault("ContentType", "application/octet-stream"));
+ value.put("TrainingInputMode", c.getOrDefault("TrainingInputMode", "File"));
+ out.put(String.valueOf(c.get("ChannelName")), value);
+ }
+ return out;
+ }
+
+ @SuppressWarnings("unchecked")
+ private void copyChannelObjects(TarArchiveOutputStream tar, Map<String, Object> channel) throws Exception {
+ String channelName = String.valueOf(channel.get("ChannelName"));
+ Object dataSource = channel.get("DataSource");
+ if (!(dataSource instanceof Map<?, ?> ds)) {
+ return;
+ }
+ Object s3 = ds.get("S3DataSource");
+ if (!(s3 instanceof Map<?, ?> s3ds)) {
+ return;
+ }
+ String s3Uri = String.valueOf(s3ds.get("S3Uri"));
+ S3Uri uri = S3Uri.parse(s3Uri);
+ List<S3Object> objects = uri.key().isBlank()
+ ? s3Service.listObjects(uri.bucket(), "", null, 1000)
+ : s3Service.listObjects(uri.bucket(), uri.key(), null, 1000);
+ for (S3Object object : objects) {
+ S3Object full = s3Service.getObject(uri.bucket(), object.getKey());
+ String relative = relativeToPrefix(uri.key(), object.getKey());
+ String safeRelative = safeRelativePath(relative);
+ if (safeRelative != null && !safeRelative.isBlank()) {
+ SageMakerEndpointManager.addFile(tar, "opt/ml/input/data/" + channelName + "/" + safeRelative, full.getData());
+ }
+ }
+ }
+
+ /**
+ * The part of an object key below the channel's S3 prefix, preserving any subdirectory
+ * structure instead of flattening every object to its basename (which loses structure and
+ * collides same-named objects in different "folders" under the prefix).
+ */
+ private static String relativeToPrefix(String prefix, String key) {
+ if (prefix.isBlank()) {
+ return key;
+ }
+ String folder = prefix.endsWith("/") ? prefix : prefix + "/";
+ if (key.startsWith(folder)) {
+ return key.substring(folder.length());
+ }
+ // The prefix matched key without a folder boundary (e.g. prefix "input" matching
+ // "input.csv" itself, or a same-named sibling key equal to the prefix): fall back to
+ // the basename rather than a key that still carries the shared prefix.
+ return key.equals(prefix) || key.startsWith(prefix)
+ ? key.substring(key.lastIndexOf('/') + 1)
+ : key;
+ }
+
+ /** Rejects an absolute path or one that escapes the channel directory via "..". */
+ private static String safeRelativePath(String relative) {
+ Path normalized = Path.of(relative).normalize();
+ if (normalized.isAbsolute() || normalized.startsWith("..") || normalized.toString().equals("..")) {
+ return null;
+ }
+ return normalized.toString();
+ }
+
+ private Integer waitForExit(String containerId, Duration timeout) throws InterruptedException {
+ long deadline = timeout.isZero() ? Long.MAX_VALUE : System.currentTimeMillis() + timeout.toMillis();
+ while (System.currentTimeMillis() < deadline) {
+ Integer exit = getExitCodeIfStopped(containerId);
+ if (exit != null) {
+ return exit;
+ }
+ Thread.sleep(500);
+ }
+ return null;
+ }
+
+ private Integer getExitCodeIfStopped(String containerId) {
+ try {
+ InspectContainerResponse inspect = lifecycleManager.getDockerClient().inspectContainerCmd(containerId).exec();
+ if (Boolean.TRUE.equals(inspect.getState().getRunning())) {
+ return null;
+ }
+ Long exit = inspect.getState().getExitCodeLong();
+ return exit == null ? 0 : exit.intValue();
+ } catch (NotFoundException e) {
+ return 1;
+ }
+ }
+
+ private Duration timeout(TrainingJobResource job) {
+ Object value = job.stoppingCondition.get("MaxRuntimeInSeconds");
+ if (value instanceof Number n) {
+ return Duration.ofSeconds(n.longValue());
+ }
+ return Duration.ZERO;
+ }
+
+ private String uploadModelArtifacts(String containerId, TrainingJobResource job) throws Exception {
+ byte[] data = modelTarGz(containerId);
+ S3Uri out = S3Uri.parse(String.valueOf(job.outputDataConfig.get("S3OutputPath")));
+ String prefix = out.key().isBlank() ? "" : (out.key().endsWith("/") ? out.key() : out.key() + "/");
+ String key = prefix + job.trainingJobName + "/output/model.tar.gz";
+ s3Service.putObject(out.bucket(), key, data, "application/x-tar", Map.of());
+ return "s3://" + out.bucket() + "/" + key;
+ }
+
+ private byte[] modelTarGz(String containerId) throws Exception {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (GzipCompressorOutputStream gz = new GzipCompressorOutputStream(out);
+ TarArchiveOutputStream tarOut = new TarArchiveOutputStream(gz);
+ InputStream dockerTar = lifecycleManager.getDockerClient().copyArchiveFromContainerCmd(containerId, "/opt/ml/model").exec();
+ TarArchiveInputStream tarIn = new TarArchiveInputStream(dockerTar)) {
+ TarArchiveEntry e;
+ String prefix = null;
+ while ((e = tarIn.getNextEntry()) != null) {
+ if (!tarIn.canReadEntryData(e) || e.isDirectory()) continue;
+ String name = e.getName();
+ if (prefix == null) {
+ int slash = name.indexOf('/');
+ prefix = slash >= 0 ? name.substring(0, slash + 1) : "";
+ }
+ if (!prefix.isBlank() && name.startsWith(prefix)) {
+ name = name.substring(prefix.length());
+ }
+ if (!name.isBlank()) {
+ SageMakerEndpointManager.addFile(tarOut, name, tarIn.readAllBytes());
+ }
+ }
+ }
+ return out.toByteArray();
+ }
+
+ private String readFailure(String containerId, int exitCode) {
+ StringBuilder sb = new StringBuilder();
+ try {
+ lifecycleManager.getDockerClient().logContainerCmd(containerId)
+ .withStdErr(true).withTail(20)
+ .exec(new LogContainerResultCallback() {
+ @Override
+ public void onNext(Frame item) {
+ sb.append(new String(item.getPayload(), StandardCharsets.UTF_8));
+ }
+ }).awaitCompletion(Duration.ofSeconds(2).toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ LOG.warnv("Interrupted while reading SageMaker training failure log for container {0}: {1}", containerId, e.getMessage());
+ Thread.currentThread().interrupt();
+ return "Container exited with code " + exitCode;
+ } catch (Exception e) {
+ LOG.warnv("Could not read SageMaker training failure log for container {0}: {1}", containerId, e.getMessage());
+ }
+ String detail = sb.toString().trim();
+ return detail.isBlank() ? "Container exited with code " + exitCode : detail;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/model/SageMakerEntities.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/model/SageMakerEntities.java
new file mode 100644
index 0000000000..57482a3ad4
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/model/SageMakerEntities.java
@@ -0,0 +1,88 @@
+package io.github.hectorvent.floci.services.sagemaker.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+@RegisterForReflection
+public final class SageMakerEntities {
+ private SageMakerEntities() {
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class ModelResource {
+ public String modelName;
+ public String modelArn;
+ public Map<String, Object> primaryContainer = new LinkedHashMap<>();
+ public List<Map<String, Object>> containers = new ArrayList<>();
+ public String executionRoleArn;
+ public long creationTime;
+ public String region;
+ public String accountId;
+ public Map<String, String> tags = new LinkedHashMap<>();
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class EndpointConfigResource {
+ public String endpointConfigName;
+ public String endpointConfigArn;
+ public List<Map<String, Object>> productionVariants = new ArrayList<>();
+ public long creationTime;
+ public String region;
+ public String accountId;
+ public Map<String, String> tags = new LinkedHashMap<>();
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class EndpointResource {
+ public String endpointName;
+ public String endpointArn;
+ public String endpointConfigName;
+ public String endpointStatus;
+ public String failureReason;
+ public long creationTime;
+ public long lastModifiedTime;
+ public String region;
+ public String accountId;
+ public String containerId;
+ public String invokeHost;
+ public int invokePort;
+ // Bumped every time a start (create or update) is kicked off; a start worker only
+ // persists its result while it is still the current one for this endpoint, so a
+ // superseding update or a delete does not get overwritten by a stale worker finishing
+ // late.
+ public long generation;
+ public Map<String, String> tags = new LinkedHashMap<>();
+ }
+
+ @RegisterForReflection
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class TrainingJobResource {
+ public String trainingJobName;
+ public String trainingJobArn;
+ public String trainingJobStatus;
+ public String secondaryStatus;
+ public String failureReason;
+ public long creationTime;
+ public long trainingStartTime;
+ public long trainingEndTime;
+ public String region;
+ public String accountId;
+ public Map<String, Object> algorithmSpecification = new LinkedHashMap<>();
+ public List<Map<String, Object>> inputDataConfig = new ArrayList<>();
+ public Map<String, Object> outputDataConfig = new LinkedHashMap<>();
+ public Map<String, Object> resourceConfig = new LinkedHashMap<>();
+ public Map<String, Object> stoppingCondition = new LinkedHashMap<>();
+ public Map<String, String> hyperParameters = new LinkedHashMap<>();
+ public String modelArtifactsS3ModelArtifacts;
+ public String containerId;
+ public Map<String, String> tags = new LinkedHashMap<>();
+ }
+}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index d738af19bf..3f272adce3 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -165,6 +165,8 @@ floci:
flush-interval-ms: 5000
opensearch:
flush-interval-ms: 5000
+ sagemaker:
+ flush-interval-ms: 5000
batch:
flush-interval-ms: 5000
lightsail:
@@ -600,6 +602,9 @@ floci:
codebuild:
enabled: true
# docker-network: floci-network
+ sagemaker:
+ enabled: true
+ # docker-network: floci-network
batch:
enabled: true
runner-mode: immediate
diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java
new file mode 100644
index 0000000000..b9ff351f87
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java
@@ -0,0 +1,182 @@
+package io.github.hectorvent.floci.services.sagemaker;
+
+import com.github.dockerjava.api.DockerClient;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.Map;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+@QuarkusTest
+class SageMakerDockerIntegrationTest {
+ private static final String AUTH =
+ "AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/sagemaker/aws4_request";
+
+ @Inject DockerClient dockerClient;
+ @Inject S3Service s3Service;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @BeforeEach
+ void requireDocker() {
+ Assumptions.assumeTrue(isDockerAvailable(), "Docker daemon must be available for SageMaker Docker tests");
+ }
+
+ @Test
+ void trainingJobRunsRealContainerAndUploadsModelArtifacts() throws Exception {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String bucket = "sm-train-" + suffix;
+ s3Service.createBucket(bucket, "us-east-1");
+ s3Service.putObject(bucket, "input/data.txt", "hello".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
+ String job = "sm-train-" + suffix;
+ post("SageMaker.CreateTrainingJob", """
+ {
+ "TrainingJobName":"%s",
+ "AlgorithmSpecification":{
+ "TrainingImage":"public.ecr.aws/docker/library/busybox:stable",
+ "ContainerEntrypoint":["/bin/sh","-c"],
+ "ContainerArguments":["mkdir -p /opt/ml/model && cp /opt/ml/input/data/train/data.txt /opt/ml/model/model.txt"]
+ },
+ "InputDataConfig":[{"ChannelName":"train","DataSource":{"S3DataSource":{"S3Uri":"s3://%s/input"}},"ContentType":"text/plain","TrainingInputMode":"File"}],
+ "OutputDataConfig":{"S3OutputPath":"s3://%s/output"},
+ "ResourceConfig":{"InstanceType":"ml.m5.large","InstanceCount":1,"VolumeSizeInGB":1},
+ "StoppingCondition":{"MaxRuntimeInSeconds":60}
+ }
+ """.formatted(job, bucket, bucket)).then().statusCode(200).body("TrainingJobArn", notNullValue());
+
+ waitForTraining(job, "Completed");
+ String artifact = post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job))
+ .then().statusCode(200).body("TrainingJobStatus", equalTo("Completed"))
+ .extract().path("ModelArtifacts.S3ModelArtifacts");
+ S3Uri uri = S3Uri.parse(artifact);
+ assertNotNull(s3Service.getObject(uri.bucket(), uri.key()).getData());
+ }
+
+ @Test
+ void endpointHostsRealContainerAndRuntimeProxiesInvocations() throws Exception {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String model = "sm-ep-model-" + suffix;
+ String cfg = "sm-ep-cfg-" + suffix;
+ String endpoint = "sm-endpoint-" + suffix;
+ String script = "cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200 if self.path == '/ping' else 404); self.end_headers()\n def do_POST(self):\n n=int(self.headers.get('content-length','0')); b=self.rfile.read(n); self.send_response(200); self.send_header('Content-Type','text/plain'); self.end_headers(); self.wfile.write(b.upper())\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py";
+ post("SageMaker.CreateModel", """
+ {"ModelName":"%s","PrimaryContainer":{"Image":"public.ecr.aws/docker/library/python:3-alpine","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}
+ """.formatted(model, json(script))).then().statusCode(200);
+ post("SageMaker.CreateEndpointConfig", """
+ {"EndpointConfigName":"%s","ProductionVariants":[{"VariantName":"AllTraffic","ModelName":"%s","InitialInstanceCount":1,"InstanceType":"ml.t2.medium","InitialVariantWeight":1.0}]}
+ """.formatted(cfg, model)).then().statusCode(200);
+ post("SageMaker.CreateEndpoint", "{\"EndpointName\":\"%s\",\"EndpointConfigName\":\"%s\"}".formatted(endpoint, cfg))
+ .then().statusCode(200);
+ waitForEndpoint(endpoint, "InService");
+ given().header("Authorization", "AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/runtime.sagemaker/aws4_request")
+ .contentType("text/plain").body("hello")
+ .when().post("/endpoints/%s/invocations".formatted(endpoint))
+ .then().statusCode(200).body(equalTo("HELLO"));
+ post("SageMaker.DeleteEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(200);
+ }
+
+ @Test
+ void stoppingATrainingJobDoesNotGetRelabeledFailed() throws Exception {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String bucket = "sm-stop-" + suffix;
+ s3Service.createBucket(bucket, "us-east-1");
+ String job = "sm-stop-" + suffix;
+ post("SageMaker.CreateTrainingJob", """
+ {
+ "TrainingJobName":"%s",
+ "AlgorithmSpecification":{
+ "TrainingImage":"public.ecr.aws/docker/library/busybox:stable",
+ "ContainerEntrypoint":["/bin/sh","-c"],
+ "ContainerArguments":["sleep 60"]
+ },
+ "OutputDataConfig":{"S3OutputPath":"s3://%s/output"},
+ "ResourceConfig":{"InstanceType":"ml.m5.large","InstanceCount":1,"VolumeSizeInGB":1},
+ "StoppingCondition":{"MaxRuntimeInSeconds":300}
+ }
+ """.formatted(job, bucket)).then().statusCode(200);
+ // Give the container time to actually be created and start running "sleep 60" so the
+ // stop races a real running container rather than one still being staged.
+ Thread.sleep(3000);
+ post("SageMaker.StopTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job)).then().statusCode(200);
+ post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job))
+ .then().statusCode(200).body("TrainingJobStatus", equalTo("Stopped"));
+ // The runner's exit-code handling races the container disappearing after the stop;
+ // give it a beat and confirm it did not relabel the stop a failure once it observes that.
+ Thread.sleep(3000);
+ post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job))
+ .then().statusCode(200).body("TrainingJobStatus", equalTo("Stopped"));
+ }
+
+ @Test
+ void deletingAnEndpointDuringStartupDoesNotResurrectIt() throws Exception {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String model = "sm-del-model-" + suffix;
+ String cfg = "sm-del-cfg-" + suffix;
+ String endpoint = "sm-del-endpoint-" + suffix;
+ // Sleeps before serving so the /ping health check keeps CreateEndpoint's async start
+ // in flight well past DeleteEndpoint returning.
+ String script = "sleep 5 && cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200); self.end_headers()\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py";
+ post("SageMaker.CreateModel", """
+ {"ModelName":"%s","PrimaryContainer":{"Image":"public.ecr.aws/docker/library/python:3-alpine","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}
+ """.formatted(model, json(script))).then().statusCode(200);
+ post("SageMaker.CreateEndpointConfig", """
+ {"EndpointConfigName":"%s","ProductionVariants":[{"VariantName":"AllTraffic","ModelName":"%s","InitialInstanceCount":1,"InstanceType":"ml.t2.medium","InitialVariantWeight":1.0}]}
+ """.formatted(cfg, model)).then().statusCode(200);
+ post("SageMaker.CreateEndpoint", "{\"EndpointName\":\"%s\",\"EndpointConfigName\":\"%s\"}".formatted(endpoint, cfg))
+ .then().statusCode(200);
+ post("SageMaker.DeleteEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(200);
+ // Let the superseded start worker finish (it should discard its result, not persist it).
+ Thread.sleep(10000);
+ post("SageMaker.DescribeEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(400);
+ }
+
+ private void waitForTraining(String job, String status) throws Exception {
+ long deadline = System.currentTimeMillis() + Duration.ofSeconds(120).toMillis();
+ while (System.currentTimeMillis() < deadline) {
+ String current = post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job)).then().statusCode(200).extract().path("TrainingJobStatus");
+ if (status.equals(current)) return;
+ if ("Failed".equals(current)) throw new AssertionError("Training failed");
+ Thread.sleep(1000);
+ }
+ throw new AssertionError("Timed out waiting for training job");
+ }
+
+ private void waitForEndpoint(String endpoint, String status) throws Exception {
+ long deadline = System.currentTimeMillis() + Duration.ofSeconds(150).toMillis();
+ while (System.currentTimeMillis() < deadline) {
+ String current = post("SageMaker.DescribeEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(200).extract().path("EndpointStatus");
+ if (status.equals(current)) return;
+ if ("Failed".equals(current)) throw new AssertionError("Endpoint failed");
+ Thread.sleep(1000);
+ }
+ throw new AssertionError("Timed out waiting for endpoint");
+ }
+
+ private io.restassured.response.Response post(String target, String body) {
+ return given().header("Authorization", AUTH).header("X-Amz-Target", target)
+ .contentType("application/x-amz-json-1.1").body(body).when().post("/");
+ }
+
+ private boolean isDockerAvailable() {
+ try { dockerClient.pingCmd().exec(); return true; } catch (Exception e) { return false; }
+ }
+
+ private String json(String value) {
+ return "\"" + value.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n") + "\"";
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerIntegrationTest.java
new file mode 100644
index 0000000000..a2a2189695
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerIntegrationTest.java
@@ -0,0 +1,67 @@
+package io.github.hectorvent.floci.services.sagemaker;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+
+@QuarkusTest
+class SageMakerIntegrationTest {
+ private static final String AUTH =
+ "AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/sagemaker/aws4_request";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void jsonControlPlaneModelEndpointConfigAndTags() {
+ String suffix = Long.toString(System.nanoTime(), 36);
+ String model = "sm-model-" + suffix;
+ String arn = post("SageMaker.CreateModel", """
+ {"ModelName":"%s","PrimaryContainer":{"Image":"busybox:stable"},"Tags":[{"Key":"env","Value":"test"}]}
+ """.formatted(model))
+ .then().statusCode(200).body("ModelArn", notNullValue()).extract().path("ModelArn");
+
+ post("SageMaker.DescribeModel", "{\"ModelName\":\"%s\"}".formatted(model))
+ .then().statusCode(200).body("ModelName", equalTo(model)).body("PrimaryContainer.Image", equalTo("busybox:stable"));
+
+ String cfg = "sm-cfg-" + suffix;
+ post("SageMaker.CreateEndpointConfig", """
+ {"EndpointConfigName":"%s","ProductionVariants":[{"VariantName":"AllTraffic","ModelName":"%s","InitialInstanceCount":1,"InstanceType":"ml.t2.medium","InitialVariantWeight":1.0}]}
+ """.formatted(cfg, model))
+ .then().statusCode(200).body("EndpointConfigArn", notNullValue());
+ post("SageMaker.ListEndpointConfigs", "{}").then().statusCode(200)
+ .body("EndpointConfigs.find { it.EndpointConfigName == '%s' }.EndpointConfigName".formatted(cfg), equalTo(cfg));
+
+ post("SageMaker.AddTags", "{\"ResourceArn\":\"%s\",\"Tags\":[{\"Key\":\"owner\",\"Value\":\"sdk\"}]}".formatted(arn))
+ .then().statusCode(200).body("Tags", hasSize(2));
+ post("SageMaker.DeleteTags", "{\"ResourceArn\":\"%s\",\"TagKeys\":[\"env\"]}".formatted(arn))
+ .then().statusCode(200);
+ post("SageMaker.ListTags", "{\"ResourceArn\":\"%s\"}".formatted(arn))
+ .then().statusCode(200).body("Tags", hasSize(1)).body("Tags[0].Key", equalTo("owner"));
+
+ post("SageMaker.DeleteEndpointConfig", "{\"EndpointConfigName\":\"%s\"}".formatted(cfg)).then().statusCode(200);
+ post("SageMaker.DeleteModel", "{\"ModelName\":\"%s\"}".formatted(model)).then().statusCode(200);
+ }
+
+ @Test
+ void trainingJobValidationErrorsUseAwsJsonErrorShape() {
+ post("SageMaker.CreateTrainingJob", "{\"TrainingJobName\":\"bad\"}")
+ .then().statusCode(400).body("__type", equalTo("ValidationException"));
+ }
+
+ private io.restassured.response.Response post(String target, String body) {
+ return given().header("Authorization", AUTH)
+ .header("X-Amz-Target", target)
+ .contentType("application/x-amz-json-1.1")
+ .body(body)
+ .when().post("/");
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerServiceTest.java
new file mode 100644
index 0000000000..4b5aae4a1c
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerServiceTest.java
@@ -0,0 +1,171 @@
+package io.github.hectorvent.floci.services.sagemaker;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointConfigResource;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.ModelResource;
+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+class SageMakerServiceTest {
+ private final ObjectMapper mapper = new ObjectMapper();
+
+ @Test
+ void modelCrudAndDuplicateValidation() throws Exception {
+ SageMakerService service = service();
+ service.createModel(mapper.readTree("""
+ {"ModelName":"m1","PrimaryContainer":{"Image":"busybox:stable"},"ExecutionRoleArn":"arn:aws:iam::000000000000:role/r"}
+ """), "us-east-1");
+ assertEquals("m1", service.describeModel(mapper.readTree("{\"ModelName\":\"m1\"}"), "us-east-1").path("ModelName").asText());
+ AwsException duplicate = assertThrows(AwsException.class, () -> service.createModel(mapper.readTree("""
+ {"ModelName":"m1","PrimaryContainer":{"Image":"busybox:stable"}}
+ """), "us-east-1"));
+ assertEquals("ValidationException", duplicate.getErrorCode());
+ service.deleteModel(mapper.readTree("{\"ModelName\":\"m1\"}"), "us-east-1");
+ AwsException missing = assertThrows(AwsException.class,
+ () -> service.describeModel(mapper.readTree("{\"ModelName\":\"m1\"}"), "us-east-1"));
+ assertEquals("ValidationException", missing.getErrorCode());
+ AwsException deleteMissing = assertThrows(AwsException.class,
+ () -> service.deleteModel(mapper.readTree("{\"ModelName\":\"m1\"}"), "us-east-1"));
+ assertEquals("ValidationException", deleteMissing.getErrorCode());
+ }
+
+ @Test
+ void modelIsScopedByRegion() throws Exception {
+ SageMakerService service = service();
+ service.createModel(mapper.readTree("""
+ {"ModelName":"m1","PrimaryContainer":{"Image":"busybox:stable"}}
+ """), "us-east-1");
+ AwsException missingInOtherRegion = assertThrows(AwsException.class,
+ () -> service.describeModel(mapper.readTree("{\"ModelName\":\"m1\"}"), "us-west-2"));
+ assertEquals("ValidationException", missingInOtherRegion.getErrorCode());
+ // Same name, different region: not a collision, since the ARNs are regional.
+ service.createModel(mapper.readTree("""
+ {"ModelName":"m1","PrimaryContainer":{"Image":"busybox:stable"}}
+ """), "us-west-2");
+ assertEquals(1, service.listModels(mapper.readTree("{}"), "us-east-1").path("Models").size());
+ assertEquals(1, service.listModels(mapper.readTree("{}"), "us-west-2").path("Models").size());
+ }
+
+ @Test
+ void createModelRejectsEmptyContainers() throws Exception {
+ SageMakerService service = service();
+ AwsException empty = assertThrows(AwsException.class, () -> service.createModel(mapper.readTree("""
+ {"ModelName":"m1","Containers":[]}
+ """), "us-east-1"));
+ assertEquals("ValidationException", empty.getErrorCode());
+ }
+
+ @Test
+ void listModelsValidatesMaxResultsAndPaginates() throws Exception {
+ SageMakerService service = service();
+ for (String name : List.of("a", "b", "c")) {
+ service.createModel(mapper.readTree(
+ "{\"ModelName\":\"" + name + "\",\"PrimaryContainer\":{\"Image\":\"busybox:stable\"}}"), "us-east-1");
+ }
+ AwsException badMax = assertThrows(AwsException.class,
+ () -> service.listModels(mapper.readTree("{\"MaxResults\":0}"), "us-east-1"));
+ assertEquals("ValidationException", badMax.getErrorCode());
+
+ JsonNode firstPage = service.listModels(mapper.readTree("{\"MaxResults\":2}"), "us-east-1");
+ assertEquals(2, firstPage.path("Models").size());
+ assertEquals("a", firstPage.path("Models").get(0).path("ModelName").asText());
+ String nextToken = firstPage.path("NextToken").asText();
+ assertEquals("b", nextToken);
+
+ JsonNode secondPage = service.listModels(
+ mapper.readTree("{\"MaxResults\":2,\"NextToken\":\"" + nextToken + "\"}"), "us-east-1");
+ assertEquals(1, secondPage.path("Models").size());
+ assertEquals("c", secondPage.path("Models").get(0).path("ModelName").asText());
+ assertTrue(secondPage.path("NextToken").isMissingNode());
+ }
+
+ @Test
+ void deleteEndpointConfigRequiresExistingConfig() throws Exception {
+ SageMakerService service = service();
+ AwsException missing = assertThrows(AwsException.class,
+ () -> service.deleteEndpointConfig(mapper.readTree("{\"EndpointConfigName\":\"missing\"}"), "us-east-1"));
+ assertEquals("ValidationException", missing.getErrorCode());
+ }
+
+ @Test
+ void endpointConfigRequiresExistingModel() throws Exception {
+ SageMakerService service = service();
+ AwsException missing = assertThrows(AwsException.class, () -> service.createEndpointConfig(mapper.readTree("""
+ {"EndpointConfigName":"cfg","ProductionVariants":[{"VariantName":"AllTraffic","ModelName":"missing","InitialInstanceCount":1,"InstanceType":"ml.t2.medium"}]}
+ """), "us-east-1"));
+ assertEquals("ValidationException", missing.getErrorCode());
+ }
+
+ @Test
+ void tagsRoundTrip() throws Exception {
+ SageMakerService service = service();
+ String arn = service.createModel(mapper.readTree("""
+ {"ModelName":"tagged","PrimaryContainer":{"Image":"busybox:stable"},"Tags":[{"Key":"a","Value":"b"}]}
+ """), "us-east-1").path("ModelArn").asText();
+ service.addTags(mapper.readTree("{\"ResourceArn\":\"" + arn + "\",\"Tags\":[{\"Key\":\"c\",\"Value\":\"d\"}]}"));
+ assertEquals(2, service.listTags(mapper.readTree("{\"ResourceArn\":\"" + arn + "\"}")).path("Tags").size());
+ service.deleteTags(mapper.readTree("{\"ResourceArn\":\"" + arn + "\",\"TagKeys\":[\"a\"]}"));
+ assertEquals("c", service.listTags(mapper.readTree("{\"ResourceArn\":\"" + arn + "\"}")).path("Tags").get(0).path("Key").asText());
+ }
+
+ @Test
+ void tagsWithoutAKeyAreSkippedInsteadOfStoredWithANullKey() throws Exception {
+ SageMakerService service = service();
+ String arn = service.createModel(mapper.readTree("""
+ {"ModelName":"m1","PrimaryContainer":{"Image":"busybox:stable"},"Tags":[{"Value":"orphan"},{"Key":"a","Value":"b"}]}
+ """), "us-east-1").path("ModelArn").asText();
+ JsonNode tags = service.listTags(mapper.readTree("{\"ResourceArn\":\"" + arn + "\"}")).path("Tags");
+ assertEquals(1, tags.size());
+ assertEquals("a", tags.get(0).path("Key").asText());
+ }
+
+ @Test
+ void trainingJobListingFiltersByNameAndStatus() throws Exception {
+ SageMakerService service = service();
+ for (String name : List.of("job-a", "job-b", "other")) {
+ service.createTrainingJob(mapper.readTree("""
+ {"TrainingJobName":"%s",
+ "AlgorithmSpecification":{"TrainingImage":"busybox:stable"},
+ "OutputDataConfig":{"S3OutputPath":"s3://bucket/out"}}
+ """.formatted(name)), "us-east-1");
+ }
+ JsonNode filtered = service.listTrainingJobs(mapper.readTree("{\"NameContains\":\"job-\"}"), "us-east-1");
+ assertEquals(2, filtered.path("TrainingJobSummaries").size());
+
+ JsonNode byStatus = service.listTrainingJobs(mapper.readTree("{\"StatusEquals\":\"Completed\"}"), "us-east-1");
+ assertEquals(0, byStatus.path("TrainingJobSummaries").size());
+ }
+
+
+ @Test
+ void s3UriParsingCoversPrefixesAndValidation() {
+ S3Uri object = S3Uri.parse("s3://bucket/path/to/object");
+ assertEquals("bucket", object.bucket());
+ assertEquals("path/to/object", object.key());
+ S3Uri bucketOnly = S3Uri.parse("s3://bucket");
+ assertEquals("bucket", bucketOnly.bucket());
+ assertEquals("", bucketOnly.key());
+ assertThrows(IllegalArgumentException.class, () -> S3Uri.parse("http://bucket/key"));
+ }
+
+ private SageMakerService service() {
+ return new SageMakerService(new InMemoryStorage<String, ModelResource>(),
+ new InMemoryStorage<String, EndpointConfigResource>(),
+ new InMemoryStorage<String, EndpointResource>(),
+ new InMemoryStorage<String, TrainingJobResource>(),
+ new RegionResolver("us-east-1", "000000000000"), mapper,
+ mock(SageMakerEndpointManager.class), mock(SageMakerTrainingRunner.class));
+ }
+}
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index 010a662f05..41858aa365 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -44,6 +44,8 @@ floci:
flush-interval-ms: 60000
opensearch:
flush-interval-ms: 60000
+ sagemaker:
+ flush-interval-ms: 60000
batch:
flush-interval-ms: 60000
lightsail:
@@ -315,6 +317,9 @@ floci:
codebuild:
enabled: true
# docker-network: floci-network
+ sagemaker:
+ enabled: true
+ # docker-network: floci-network
batch:
enabled: true
runner-mode: immediate
diff --git a/tools/docs/services.yaml b/tools/docs/services.yaml
index ca8496125c..aaa5622b7d 100644
--- a/tools/docs/services.yaml
+++ b/tools/docs/services.yaml
@@ -310,6 +310,10 @@ services:
doc: docs/services/route53resolver.md
sources:
- { path: src/main/java/io/github/hectorvent/floci/services/route53resolver/Route53ResolverJsonHandler.java, mode: switch }
+ - service: sagemaker
+ doc: docs/services/sagemaker.md
+ sources:
+ - { path: src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerJsonHandler.java, mode: switch }
- service: securityhub
doc: docs/services/securityhub.md
sources: