Light, fluffy, and always free - The AWS Local Emulator alternative
feat(sagemaker): add SageMaker emulation with Docker-backed training (#1949)
Marcus Bergoe2aefa654bbf
diff --git a/compatibility-tests/sdk-test-python/tests/test_sagemaker.py b/compatibility-tests/sdk-test-python/tests/test_sagemaker.pynew file mode 100644index 0000000000..2c3cc99965--- /dev/null+++ b/compatibility-tests/sdk-test-python/tests/test_sagemaker.py@@ -0,0 +1,74 @@+import os+import time+import uuid++import boto3+import pytest+from botocore.config import Config+++def client(service):+ return boto3.client(+ service,+ # Every other test file gets this from the conftest.py fixtures, which read+ # FLOCI_ENDPOINT (the test runner container's compat-net hostname for floci, e.g.+ # http://floci:4566). This file has no fixture of its own, so it must read the same+ # variable directly instead of hardcoding localhost, which only resolves to floci+ # when running the suite straight on the host rather than in its own CI container.+ endpoint_url=os.environ.get("FLOCI_ENDPOINT", "http://localhost:4566"),+ region_name="us-east-1",+ aws_access_key_id="test",+ aws_secret_access_key="test",+ config=Config(retries={"max_attempts": 1}),+ )+++def test_sagemaker_control_plane_and_training():+ sm = client("sagemaker")+ s3 = client("s3")+ suffix = uuid.uuid4().hex[:8]+ model = f"compat-sm-{suffix}"+ cfg = f"compat-sm-cfg-{suffix}"+ bucket = f"compat-sm-{suffix}"++ sm.create_model(ModelName=model, PrimaryContainer={"Image": "public.ecr.aws/docker/library/busybox:stable"}, ExecutionRoleArn="arn:aws:iam::000000000000:role/r")+ assert sm.describe_model(ModelName=model)["ModelName"] == model+ sm.create_endpoint_config(EndpointConfigName=cfg, ProductionVariants=[{"VariantName": "AllTraffic", "ModelName": model, "InitialInstanceCount": 1, "InstanceType": "ml.t2.medium"}])+ assert any(c["EndpointConfigName"] == cfg for c in sm.list_endpoint_configs()["EndpointConfigs"])++ s3.create_bucket(Bucket=bucket)+ s3.put_object(Bucket=bucket, Key="input/data.txt", Body=b"hello")+ job = f"compat-train-{suffix}"+ sm.create_training_job(+ TrainingJobName=job,+ RoleArn="arn:aws:iam::000000000000:role/r",+ AlgorithmSpecification={+ "TrainingImage": "public.ecr.aws/docker/library/busybox:stable",+ "TrainingInputMode": "File",+ "ContainerEntrypoint": ["/bin/sh", "-c"],+ "ContainerArguments": ["mkdir -p /opt/ml/model && echo ok > /opt/ml/model/model.txt"],+ },+ InputDataConfig=[{+ "ChannelName": "train",+ "DataSource": {"S3DataSource": {"S3DataType": "S3Prefix", "S3Uri": f"s3://{bucket}/input"}},+ }],+ OutputDataConfig={"S3OutputPath": f"s3://{bucket}/output"},+ ResourceConfig={"InstanceType": "ml.m5.large", "InstanceCount": 1, "VolumeSizeInGB": 1},+ StoppingCondition={"MaxRuntimeInSeconds": 60},+ )+ deadline = time.time() + 120+ while time.time() < deadline:+ desc = sm.describe_training_job(TrainingJobName=job)+ if desc["TrainingJobStatus"] == "Completed":+ artifact = desc["ModelArtifacts"]["S3ModelArtifacts"]+ key = artifact.split(f"s3://{bucket}/", 1)[1]+ assert s3.get_object(Bucket=bucket, Key=key)["Body"].read()+ break+ if desc["TrainingJobStatus"] == "Failed":+ pytest.fail(desc.get("FailureReason", "training failed"))+ time.sleep(1)+ else:+ pytest.fail("Timed out waiting for SageMaker training job")++ sm.delete_endpoint_config(EndpointConfigName=cfg)+ sm.delete_model(ModelName=model)diff --git a/docs/services/index.md b/docs/services/index.mdindex 1a80506a4d..557a92c61f 100644--- a/docs/services/index.md+++ b/docs/services/index.md@@ -120,6 +120,8 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac| [CloudFront](cloudfront.md) | `/2020-05-31/distribution/*`, `/2020-05-31/cache-policy/*`, `/2020-05-31/function/*` | REST XML | 50 || [Route53](route53.md) | `/2013-04-01/hostedzone/*`, `/2013-04-01/healthcheck/*`, `/2013-04-01/change/*` | REST XML | 25 || [Route 53 Resolver](route53resolver.md) | `POST /` + `X-Amz-Target: Route53Resolver.*` | JSON 1.1 | 18 |+| [Amazon SageMaker](sagemaker.md) | `POST /` + `X-Amz-Target: SageMaker.*` | JSON 1.1 | 20 |+| [SageMaker Runtime](sagemaker.md#endpoint-hosting) | `/endpoints/{EndpointName}/invocations` | REST (binary payload) | 1 || [Cloud Map](cloudmap.md) | `POST /` + `X-Amz-Target: Route53AutoNaming_v20170314.*` | JSON 1.1 | 22 || [AWS Config](config.md) | `POST /` + `X-Amz-Target: StarlingDoveService.*` | JSON 1.1 | 33 || [CloudTrail](cloudtrail.md) | `POST /` + `X-Amz-Target: com.amazonaws.cloudtrail.v20131101.CloudTrail_20131101.*` | JSON 1.1 | 9 |@@ -154,3 +156,5 @@ export AWS_SECRET_ACCESS_KEY=test````AWS_ENDPOINT_URL` is the standard env var recognised by the AWS CLI v2 and AWS SDKs v2+, so no `--endpoint-url` flag is needed on each command.++- [SageMaker](sagemaker.md) - model, endpoint, runtime, and training emulation with Docker execution.diff --git a/docs/services/sagemaker.md b/docs/services/sagemaker.mdnew file mode 100644index 0000000000..ddba8dd45a--- /dev/null+++ b/docs/services/sagemaker.md@@ -0,0 +1,63 @@+# Amazon SageMaker++Floci emulates SageMaker control-plane APIs and runs real Docker containers for local training jobs and hosted endpoints. Containers use the SageMaker `/opt/ml` contract rather than mocks.++## Supported operations++| Area | Operations |+| --- | --- |+| Models | `CreateModel`, `DescribeModel`, `DeleteModel`, `ListModels` |+| Endpoint configs | `CreateEndpointConfig`, `DescribeEndpointConfig`, `DeleteEndpointConfig`, `ListEndpointConfigs` |+| Endpoints | `CreateEndpoint`, `DescribeEndpoint`, `UpdateEndpoint`, `DeleteEndpoint`, `ListEndpoints` |+| Training | `CreateTrainingJob`, `DescribeTrainingJob`, `ListTrainingJobs`, `StopTrainingJob` |+| Tags | `AddTags`, `ListTags`, `DeleteTags` |+| Runtime | `POST /endpoints/{EndpointName}/invocations` |++## Supported Actions++<!-- floci:actions:start -->+| Action | Description |+| --- | --- |+| `CreateModel` | Registers a model from `PrimaryContainer` or `Containers`; rejects an empty `Containers` array rather than dereferencing it. |+| `DescribeModel` | Returns a model by name, scoped to the calling region. |+| `DeleteModel` | Deletes a model by name, scoped to the calling region. |+| `ListModels` | Lists models for the calling region; supports `NameContains`, `MaxResults` (1-100), and `NextToken` pagination. |+| `CreateEndpointConfig` | Creates an endpoint configuration; every `ProductionVariants[].ModelName` must already exist. |+| `DescribeEndpointConfig` | Returns an endpoint configuration by name, scoped to the calling region. |+| `DeleteEndpointConfig` | Deletes an endpoint configuration by name, scoped to the calling region. |+| `ListEndpointConfigs` | Lists endpoint configurations for the calling region; supports `MaxResults` and `NextToken`. |+| `CreateEndpoint` | Starts hosting the configured model image asynchronously; the endpoint is `Creating` until the container passes its `/ping` check. |+| `DescribeEndpoint` | Returns an endpoint's status, scoped to the calling region. |+| `DeleteEndpoint` | Stops and removes the endpoint's container; a start still in flight for this endpoint discards its result instead of resurrecting the deleted record. |+| `ListEndpoints` | Lists endpoints for the calling region; supports `MaxResults` and `NextToken`. |+| `UpdateEndpoint` | Swaps an endpoint to a new endpoint configuration, restarting its container; a superseded in-flight start discards its result. |+| `CreateTrainingJob` | Runs `AlgorithmSpecification.TrainingImage` asynchronously against the `/opt/ml` contract and uploads the resulting model artifacts to S3. |+| `DescribeTrainingJob` | Returns a training job's status, scoped to the calling region. |+| `ListTrainingJobs` | Lists training jobs for the calling region; supports `NameContains`, `StatusEquals`, `MaxResults`, and `NextToken`. |+| `StopTrainingJob` | Stops a training job that is `InProgress` or `Stopping`; a no-op against a job already in a terminal state. |+| `AddTags` | Adds tags to a model, endpoint config, endpoint, or training job identified by ARN. |+| `ListTags` | Lists tags for a model, endpoint config, endpoint, or training job identified by ARN. |+| `DeleteTags` | Removes tags by key from a model, endpoint config, endpoint, or training job identified by ARN. |+<!-- floci:actions:end -->++## Training contract++`CreateTrainingJob` starts `AlgorithmSpecification.TrainingImage` with command `train` unless `ContainerEntrypoint`/`ContainerArguments` are supplied. Floci writes SageMaker config files under `/opt/ml/input/config`, downloads channel data from S3 into `/opt/ml/input/data/<channel>`, waits for container exit, and uploads `/opt/ml/model` as `model.tar.gz` under `OutputDataConfig.S3OutputPath/<TrainingJobName>/output/`.++## Endpoint hosting++`CreateEndpoint` starts the model image as a long-lived Docker container with command `serve`, port `8080`, `/ping` health checks, and `/invocations` runtime proxying. `ModelDataUrl` artifacts are downloaded from S3 and placed in `/opt/ml/model`.++## Examples++```python+import boto3+sm = boto3.client("sagemaker", endpoint_url="http://localhost:4566", region_name="us-east-1")+sm.create_model(ModelName="m", PrimaryContainer={"Image":"my-image"}, ExecutionRoleArn="arn:aws:iam::000000000000:role/r")+sm.create_endpoint_config(EndpointConfigName="cfg", ProductionVariants=[{"VariantName":"AllTraffic","ModelName":"m","InitialInstanceCount":1,"InstanceType":"ml.t2.medium"}])+sm.create_endpoint(EndpointName="ep", EndpointConfigName="cfg")+```++```bash+aws --endpoint-url=http://localhost:4566 sagemaker list-models+```diff --git a/mkdocs.yml b/mkdocs.ymlindex 03e3a04106..4e7a6be03c 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -181,6 +181,7 @@ nav:- Elastic Beanstalk: services/elastic-beanstalk.md- CodeBuild: services/codebuild.md- AWS Batch: services/batch.md+ - SageMaker: services/sagemaker.md- CodeDeploy: services/codedeploy.md- CodeGuru Reviewer: services/codegurureviewer.md- CodePipeline: services/codepipeline.mddiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 0911a51c7d..3f2602d139 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -336,6 +336,7 @@ public interface EmulatorConfig {LakeFormationStorageConfig lakeformation();EfsStorageConfig efs();+ SageMakerStorageConfig sagemaker();}interface ApsStorageConfig {@@ -611,6 +612,7 @@ public interface EmulatorConfig {@WithDefault("5000")long flushIntervalMs();}+interface EfsStorageConfig {Optional<String> mode();@@ -618,6 +620,13 @@ public interface EmulatorConfig {long flushIntervalMs();}+ interface SageMakerStorageConfig {+ Optional<String> mode();++ @WithDefault("5000")+ long flushIntervalMs();+ }+interface CodeDeployStorageConfig {Optional<String> mode();@@ -727,6 +736,7 @@ public interface EmulatorConfig {CloudFrontServiceConfig cloudfront();AppSyncServiceConfig appsync();BatchServiceConfig batch();+ SageMakerServiceConfig sagemaker();LightsailServiceConfig lightsail();UiServiceConfig ui();S3VectorsServiceConfig s3vectors();@@ -1104,6 +1114,13 @@ public interface EmulatorConfig {Optional<String> dockerNetwork();}+ interface SageMakerServiceConfig {+ @WithDefault("true")+ boolean enabled();++ Optional<String> dockerNetwork();+ }+interface CodeDeployServiceConfig {@WithDefault("true")boolean enabled();diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.javaindex 3ad4b3b93f..b943824fc0 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java@@ -57,6 +57,7 @@ import io.github.hectorvent.floci.services.route53resolver.Route53ResolverJsonHaimport io.github.hectorvent.floci.services.networkfirewall.NetworkFirewallJsonHandler;import io.github.hectorvent.floci.services.servicecatalog.ServiceCatalogJsonHandler;import io.github.hectorvent.floci.services.servicequotas.ServiceQuotasJsonHandler;+import io.github.hectorvent.floci.services.sagemaker.SageMakerJsonHandler;import io.github.hectorvent.floci.services.ssm.Ec2MessagesJsonHandler;import io.github.hectorvent.floci.services.ssm.SsmJsonHandler;import jakarta.inject.Inject;@@ -139,6 +140,7 @@ public class AwsJson11Controller {private final ServiceQuotasJsonHandler serviceQuotasJsonHandler;private final MarketplaceEntitlementController marketplaceEntitlementController;private final MarketplaceMeteringController marketplaceMeteringController;+ private final SageMakerJsonHandler sageMakerJsonHandler;@Injectpublic AwsJson11Controller(ObjectMapper objectMapper, ResolvedServiceCatalog catalog,@@ -192,7 +194,8 @@ public class AwsJson11Controller {BudgetsJsonHandler budgetsJsonHandler,ServiceQuotasJsonHandler serviceQuotasJsonHandler,MarketplaceEntitlementController marketplaceEntitlementController,- MarketplaceMeteringController marketplaceMeteringController) {+ MarketplaceMeteringController marketplaceMeteringController,+ SageMakerJsonHandler sageMakerJsonHandler) {this.objectMapper = objectMapper;this.strictBodyReader = objectMapper.reader().with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);this.catalog = catalog;@@ -251,6 +254,7 @@ public class AwsJson11Controller {this.serviceQuotasJsonHandler = serviceQuotasJsonHandler;this.marketplaceEntitlementController = marketplaceEntitlementController;this.marketplaceMeteringController = marketplaceMeteringController;+ this.sageMakerJsonHandler = sageMakerJsonHandler;}@POST@@ -362,6 +366,7 @@ public class AwsJson11Controller {}yield marketplaceMeteringController.handle(action, request, region);}+ case "sagemaker" -> sageMakerJsonHandler.handle(action, request, region);default -> null;};// catalog.matchTarget is protocol-agnostic: a JSON 1.0 targetdiff --git a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.javaindex 53e6968784..32999c1e03 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java@@ -60,6 +60,7 @@ import io.github.hectorvent.floci.services.marketplace.MarketplaceCatalogControlimport io.github.hectorvent.floci.services.marketplace.MarketplaceDeploymentController;import io.github.hectorvent.floci.services.marketplace.MarketplaceDiscoveryController;import io.github.hectorvent.floci.services.marketplace.MarketplaceReportingController;+import io.github.hectorvent.floci.services.sagemaker.SageMakerRuntimeController;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;@@ -402,6 +403,12 @@ public class ResolvedServiceCatalog {config.storage().services().batch().flushIntervalMs(), null, ServiceProtocol.REST_JSON,protocols(ServiceProtocol.REST_JSON),Set.of(), Set.of("batch"), Set.of(), Set.of(BatchController.class)),+ descriptor("sagemaker", "sagemaker", config.services().sagemaker().enabled(), true,+ "sagemaker", storageMode(config.storage().services().sagemaker().mode(), config.storage().mode()),+ config.storage().services().sagemaker().flushIntervalMs(), null, ServiceProtocol.JSON,+ protocols(ServiceProtocol.JSON, ServiceProtocol.REST_JSON),+ Set.of("SageMaker."), Set.of("sagemaker", "runtime.sagemaker"), Set.of(),+ Set.of(SageMakerRuntimeController.class)),descriptor("codedeploy", "codedeploy", config.services().codedeploy().enabled(), true,"codedeploy", storageMode(config.storage().services().codedeploy().mode(), config.storage().mode()),config.storage().services().codedeploy().flushIntervalMs(), null, ServiceProtocol.JSON,diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/S3Uri.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/S3Uri.javanew file mode 100644index 0000000000..941754dd9e--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/S3Uri.java@@ -0,0 +1,19 @@+package io.github.hectorvent.floci.services.sagemaker;++record S3Uri(String bucket, String key) {+ static S3Uri parse(String uri) {+ if (uri == null || !uri.startsWith("s3://")) {+ throw new IllegalArgumentException("Expected s3:// URI");+ }+ String rest = uri.substring(5);+ int slash = rest.indexOf('/');+ String bucket = slash < 0 ? rest : rest.substring(0, slash);+ if (bucket.isBlank()) {+ throw new IllegalArgumentException("Expected s3://<bucket>/<key>, got: " + uri);+ }+ if (slash < 0) {+ return new S3Uri(bucket, "");+ }+ return new S3Uri(bucket, rest.substring(slash + 1));+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.javanew file mode 100644index 0000000000..ea4c9e2665--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerEndpointManager.java@@ -0,0 +1,268 @@+package io.github.hectorvent.floci.services.sagemaker;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.ContainerTeardown;+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.ModelResource;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;+import org.jboss.logging.Logger;++import java.io.ByteArrayInputStream;+import java.io.ByteArrayOutputStream;+import java.net.ConnectException;+import java.net.URI;+import java.net.http.HttpTimeoutException;+import java.net.http.HttpClient;+import java.net.http.HttpRequest;+import java.net.http.HttpResponse;+import java.nio.file.Path;+import java.time.Duration;+import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;++@ApplicationScoped+public class SageMakerEndpointManager implements ContainerTeardown {+ private static final Logger LOG = Logger.getLogger(SageMakerEndpointManager.class);+ private static final int PORT = 8080;+ private static final int DEFAULT_FILE_MODE = 0644;+ private static final int PING_LOG_FREQUENCY = 15;+ private static final Duration PING_TIMEOUT = Duration.ofSeconds(120);++ private final ContainerBuilder containerBuilder;+ private final ContainerLifecycleManager lifecycleManager;+ private final EmulatorConfig config;+ private final ContainerDetector containerDetector;+ private final S3Service s3Service;+ private final ExecutorService executor = Executors.newCachedThreadPool();+ private final ConcurrentHashMap<String, String> containers = new ConcurrentHashMap<>();+ private final HttpClient httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(2)).build();++ @Inject+ public SageMakerEndpointManager(ContainerBuilder containerBuilder, ContainerLifecycleManager lifecycleManager,+ EmulatorConfig config, ContainerDetector containerDetector, S3Service s3Service) {+ this.containerBuilder = containerBuilder;+ this.lifecycleManager = lifecycleManager;+ this.config = config;+ this.containerDetector = containerDetector;+ this.s3Service = s3Service;+ }++ public void startEndpointAsync(EndpointResource endpoint, SageMakerService service) {+ executor.submit(() -> startEndpoint(endpoint, service));+ }++ private void startEndpoint(EndpointResource endpoint, SageMakerService service) {+ String containerId = null;+ try {+ ModelResource model = service.modelForEndpoint(endpoint);+ Map<String, Object> c = model.primaryContainer;+ String image = string(c.get("Image"));+ if (image == null) {+ throw new IllegalArgumentException("PrimaryContainer.Image is required");+ }+ String name = ContainerStorageHelper.dockerName(config, "floci-sagemaker-endpoint-" + endpoint.endpointName);+ lifecycleManager.removeIfExists(name);+ ContainerBuilder.Builder builder = containerBuilder.newContainer(image)+ .withName(name)+ .withEnv(environment(endpoint.region, map(c.get("Environment"))))+ .withDockerNetwork(config.services().sagemaker().dockerNetwork())+ .withHostDockerInternalOnLinux()+ .withEmbeddedDns()+ .withPortBinding(PORT, 0)+ .withLogRotation();+ applyEntrypoint(builder, c, "serve");+ ContainerSpec spec = builder.build();+ containerId = lifecycleManager.create(spec);+ String modelDataUrl = string(c.get("ModelDataUrl"));+ if (modelDataUrl != null && !modelDataUrl.isBlank()) {+ copyModelData(containerId, modelDataUrl);+ }+ ContainerInfo info = lifecycleManager.startCreated(containerId, spec);+ EndpointInfo ep = info.getEndpoint(PORT);+ endpoint.containerId = containerId;+ endpoint.invokeHost = ep.host();+ endpoint.invokePort = ep.port();+ containers.put(endpoint.endpointName, containerId);+ waitForPing(endpoint.invokeHost, endpoint.invokePort);+ endpoint.endpointStatus = "InService";+ endpoint.failureReason = null;+ finishStart(endpoint, service);+ } catch (Exception e) {+ LOG.warnv("SageMaker endpoint {0} failed: {1}", endpoint.endpointName, e.getMessage());+ if (containerId != null) {+ lifecycleManager.stopAndRemove(containerId, null);+ }+ endpoint.endpointStatus = "Failed";+ endpoint.failureReason = e.getMessage();+ finishStart(endpoint, service);+ }+ }++ /**+ * Publishes a start attempt's outcome unless a delete or a newer update has superseded it+ * while Docker was still working: {@link SageMakerService#finalizeEndpointStart} refuses the+ * write in that case, and the container this attempt just brought up (if any) is torn down+ * instead of being left running as an orphan nothing points to any more.+ */+ private void finishStart(EndpointResource endpoint, SageMakerService service) {+ if (service.finalizeEndpointStart(endpoint)) {+ return;+ }+ LOG.infov("SageMaker endpoint {0} start superseded by a delete or newer update; discarding result",+ endpoint.endpointName);+ String containerId = endpoint.containerId != null ? endpoint.containerId : containers.remove(endpoint.endpointName);+ if (containerId != null) {+ lifecycleManager.stopAndRemove(containerId, null);+ }+ }++ public void stopEndpoint(EndpointResource endpoint) {+ String containerId = endpoint.containerId != null ? endpoint.containerId : containers.remove(endpoint.endpointName);+ if (containerId != null) {+ lifecycleManager.stopAndRemove(containerId, null);+ }+ }++ @Override+ public void stopManagedContainers() {+ containers.forEach((name, id) -> lifecycleManager.stopAndRemove(id, null));+ containers.clear();+ executor.shutdownNow();+ }++ private List<String> environment(String region, Map<String, String> modelEnv) {+ List<String> env = new ArrayList<>();+ env.add("SAGEMAKER_BIND_TO_PORT=8080");+ env.add("SAGEMAKER_REGION=" + region);+ env.add("AWS_REGION=" + region);+ env.add("AWS_DEFAULT_REGION=" + region);+ env.add("AWS_ACCESS_KEY_ID=test");+ env.add("AWS_SECRET_ACCESS_KEY=test");+ env.add("AWS_SESSION_TOKEN=test");+ String endpoint = "http://" + resolveEndpointHostname() + ":" + config.port();+ env.add("AWS_ENDPOINT_URL=" + endpoint);+ env.add("FLOCI_ENDPOINT=" + endpoint);+ modelEnv.forEach((k, v) -> env.add(k + "=" + (v == null ? "" : v)));+ return env;+ }++ private String resolveEndpointHostname() {+ return containerDetector.isRunningInContainer() ? config.hostname().orElse(EmbeddedDnsServer.DEFAULT_SUFFIX) : "host.docker.internal";+ }++ static void applyEntrypoint(ContainerBuilder.Builder builder, Map<String, Object> container, String defaultCommand) {+ Object entrypoint = container.get("ContainerEntrypoint");+ Object arguments = container.get("ContainerArguments");+ if (entrypoint instanceof List<?> list && !list.isEmpty()) {+ builder.withEntrypoint(list.stream().map(String::valueOf).toList());+ if (arguments instanceof List<?> args) {+ builder.withCmd(args.stream().map(String::valueOf).toList());+ }+ } else if (arguments instanceof List<?> args && !args.isEmpty()) {+ builder.withCmd(args.stream().map(String::valueOf).toList());+ } else {+ builder.withCmd(List.of(defaultCommand));+ }+ }++ private void copyModelData(String containerId, String s3Uri) throws Exception {+ S3Uri uri = S3Uri.parse(s3Uri);+ S3Object object = s3Service.getObject(uri.bucket(), uri.key());+ byte[] tarData = tarModelData(object.getData(), uri.key());+ lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)+ .withRemotePath("/")+ .withTarInputStream(new ByteArrayInputStream(tarData))+ .exec();+ }++ private byte[] tarModelData(byte[] data, String key) throws Exception {+ ByteArrayOutputStream out = new ByteArrayOutputStream();+ try (TarArchiveOutputStream tar = new TarArchiveOutputStream(out)) {+ if (key.endsWith(".tar.gz") || key.endsWith(".tgz")) {+ try (TarArchiveInputStream in = new TarArchiveInputStream(new GzipCompressorInputStream(new ByteArrayInputStream(data)))) {+ TarArchiveEntry e;+ while ((e = in.getNextEntry()) != null) {+ if (!in.canReadEntryData(e) || e.isDirectory()) continue;+ String entryName = e.getName();+ Path normalized = Path.of(entryName).normalize();+ if (normalized.isAbsolute() || normalized.startsWith("..")) continue;+ addFile(tar, "opt/ml/model/" + normalized.toString(), in.readAllBytes());+ }+ }+ } else {+ addFile(tar, "opt/ml/model/" + key.substring(key.lastIndexOf('/') + 1), data);+ }+ }+ return out.toByteArray();+ }++ static void addFile(TarArchiveOutputStream tar, String path, byte[] data) throws Exception {+ TarArchiveEntry entry = new TarArchiveEntry(path);+ entry.setSize(data.length);+ entry.setMode(DEFAULT_FILE_MODE);+ tar.putArchiveEntry(entry);+ tar.write(data);+ tar.closeArchiveEntry();+ }++ private void waitForPing(String host, int port) throws Exception {+ long deadline = System.currentTimeMillis() + PING_TIMEOUT.toMillis();+ URI uri = URI.create("http://" + host + ":" + port + "/ping");+ int failures = 0;+ while (System.currentTimeMillis() < deadline) {+ try {+ HttpRequest req = HttpRequest.newBuilder(uri).timeout(Duration.ofSeconds(2)).GET().build();+ if (httpClient.send(req, HttpResponse.BodyHandlers.discarding()).statusCode() == 200) {+ return;+ }+ failures++;+ } catch (ConnectException | HttpTimeoutException e) {+ failures++;+ if (failures % PING_LOG_FREQUENCY == 0) {+ LOG.warnv("Still waiting for SageMaker endpoint ping after {0} failures: {1}", failures, e.getMessage());+ } else {+ LOG.debugv("Waiting for SageMaker endpoint ping: {0}", e.getMessage());+ }+ } catch (Exception e) {+ failures++;+ LOG.warnv("Waiting for SageMaker endpoint ping failed: {0}", e.getMessage());+ }+ Thread.sleep(1000);+ }+ throw new IllegalStateException("Container did not pass /ping health check");+ }++ @SuppressWarnings("unchecked")+ static Map<String, String> map(Object value) {+ if (value instanceof Map<?, ?> in) {+ LinkedHashMap<String, String> out = new LinkedHashMap<>();+ in.forEach((k, v) -> out.put(String.valueOf(k), v == null ? "" : String.valueOf(v)));+ return out;+ }+ return Map.of();+ }++ static String string(Object value) {+ return value == null ? null : String.valueOf(value);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerJsonHandler.javanew file mode 100644index 0000000000..87a724a33b--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerJsonHandler.java@@ -0,0 +1,43 @@+package io.github.hectorvent.floci.services.sagemaker;++import com.fasterxml.jackson.databind.JsonNode;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import jakarta.ws.rs.core.Response;++@ApplicationScoped+public class SageMakerJsonHandler {+ private final SageMakerService service;++ @Inject+ public SageMakerJsonHandler(SageMakerService service) {+ this.service = service;+ }++ public Response handle(String action, JsonNode request, String region) {+ Object entity = switch (action) {+ case "CreateModel" -> service.createModel(request, region);+ case "DescribeModel" -> service.describeModel(request, region);+ case "DeleteModel" -> service.deleteModel(request, region);+ case "ListModels" -> service.listModels(request, region);+ case "CreateEndpointConfig" -> service.createEndpointConfig(request, region);+ case "DescribeEndpointConfig" -> service.describeEndpointConfig(request, region);+ case "DeleteEndpointConfig" -> service.deleteEndpointConfig(request, region);+ case "ListEndpointConfigs" -> service.listEndpointConfigs(request, region);+ case "CreateEndpoint" -> service.createEndpoint(request, region);+ case "DescribeEndpoint" -> service.describeEndpoint(request, region);+ case "DeleteEndpoint" -> service.deleteEndpoint(request, region);+ case "ListEndpoints" -> service.listEndpoints(request, region);+ case "UpdateEndpoint" -> service.updateEndpoint(request, region);+ case "CreateTrainingJob" -> service.createTrainingJob(request, region);+ case "DescribeTrainingJob" -> service.describeTrainingJob(request, region);+ case "ListTrainingJobs" -> service.listTrainingJobs(request, region);+ case "StopTrainingJob" -> service.stopTrainingJob(request, region);+ case "AddTags" -> service.addTags(request);+ case "ListTags" -> service.listTags(request);+ case "DeleteTags" -> service.deleteTags(request);+ default -> throw SageMakerService.validation("Action " + action + " is not supported");+ };+ return Response.ok(entity).build();+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerRuntimeController.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerRuntimeController.javanew file mode 100644index 0000000000..97bca1c2f6--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerRuntimeController.java@@ -0,0 +1,93 @@+package io.github.hectorvent.floci.services.sagemaker;++import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;+import jakarta.inject.Inject;+import jakarta.ws.rs.Consumes;+import jakarta.ws.rs.HeaderParam;+import jakarta.ws.rs.POST;+import jakarta.ws.rs.Path;+import jakarta.ws.rs.PathParam;+import jakarta.ws.rs.Produces;+import jakarta.ws.rs.core.Context;+import jakarta.ws.rs.core.HttpHeaders;+import jakarta.ws.rs.core.MediaType;+import jakarta.ws.rs.core.Response;+import org.jboss.logging.Logger;++import java.net.URI;+import java.net.http.HttpClient;+import java.net.http.HttpRequest;+import java.net.http.HttpResponse;+import java.nio.charset.StandardCharsets;+import java.time.Duration;+import java.util.Map;++@Path("/endpoints")+public class SageMakerRuntimeController {+ private static final Logger LOG = Logger.getLogger(SageMakerRuntimeController.class);++ private final SageMakerService service;+ private final RegionResolver regionResolver;+ private final HttpClient httpClient;++ @Inject+ public SageMakerRuntimeController(SageMakerService service, RegionResolver regionResolver) {+ this.service = service;+ this.regionResolver = regionResolver;+ this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build();+ }++ @POST+ @Path("/{endpointName}/invocations")+ @Consumes(MediaType.WILDCARD)+ @Produces(MediaType.WILDCARD)+ public Response invoke(@PathParam("endpointName") String endpointName,+ @HeaderParam("Content-Type") String contentType,+ @HeaderParam("Accept") String accept,+ @Context HttpHeaders headers,+ byte[] body) {+ String region = regionResolver.resolveRegion(headers);+ EndpointResource ep = service.endpoint(region, endpointName).orElse(null);+ if (ep == null) {+ return validation("Endpoint " + endpointName + " of account " + regionResolver.getAccountId() + " not found.");+ }+ if (!"InService".equals(ep.endpointStatus) || ep.invokeHost == null) {+ return validation("Endpoint " + endpointName + " is not InService.");+ }+ try {+ HttpRequest.Builder builder = HttpRequest.newBuilder(URI.create("http://" + ep.invokeHost + ":" + ep.invokePort + "/invocations"))+ .timeout(Duration.ofSeconds(60))+ .POST(HttpRequest.BodyPublishers.ofByteArray(body == null ? new byte[0] : body));+ if (contentType != null) {+ builder.header("Content-Type", contentType);+ }+ if (accept != null) {+ builder.header("Accept", accept);+ }+ HttpResponse<byte[]> response = httpClient.send(builder.build(), HttpResponse.BodyHandlers.ofByteArray());+ // SageMaker Runtime maps model-container 5xx responses to ModelError (HTTP 424).+ if (response.statusCode() >= 500) {+ String originalMessage = new String(response.body(), StandardCharsets.UTF_8);+ return Response.status(424).type(MediaType.APPLICATION_JSON).entity(Map.of(+ "__type", "ModelError",+ "Message", "Received server error (" + response.statusCode() + ") from primary with message \"" + originalMessage + "\"",+ "OriginalStatusCode", response.statusCode(),+ "OriginalMessage", originalMessage+ )).build();+ }+ Response.ResponseBuilder out = Response.status(response.statusCode()).entity(response.body());+ response.headers().firstValue("Content-Type").ifPresent(out::type);+ return out.build();+ } catch (Exception e) {+ LOG.warnv("SageMaker endpoint invocation failed for {0}: {1}", endpointName, e.getMessage());+ return Response.status(424).type(MediaType.APPLICATION_JSON)+ .entity(Map.of("__type", "ModelError", "message", e.getMessage())).build();+ }+ }++ private Response validation(String message) {+ return Response.status(400).type(MediaType.APPLICATION_JSON)+ .entity(Map.of("__type", "ValidationError", "message", message)).build();+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerService.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerService.javanew file mode 100644index 0000000000..8962b25c64--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerService.java@@ -0,0 +1,631 @@+package io.github.hectorvent.floci.services.sagemaker;++import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointConfigResource;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.ModelResource;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;++import java.time.Clock;+import java.util.ArrayList;+import java.util.Comparator;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.Optional;+import java.util.function.BiConsumer;+import java.util.function.Function;++@ApplicationScoped+public class SageMakerService {+ static final int DEFAULT_LIMIT = 100;+ static final int MAX_RESULTS_CEILING = 100;+ private static final ObjectMapper STATIC_MAPPER = new ObjectMapper();++ private final StorageBackend<String, ModelResource> modelStore;+ private final StorageBackend<String, EndpointConfigResource> endpointConfigStore;+ private final StorageBackend<String, EndpointResource> endpointStore;+ private final StorageBackend<String, TrainingJobResource> trainingJobStore;+ private final RegionResolver regionResolver;+ private final ObjectMapper mapper;+ private final SageMakerEndpointManager endpointManager;+ private final SageMakerTrainingRunner trainingRunner;+ private final Clock clock;++ @Inject+ public SageMakerService(StorageFactory storageFactory, RegionResolver regionResolver, ObjectMapper mapper,+ SageMakerEndpointManager endpointManager, SageMakerTrainingRunner trainingRunner) {+ this(storageFactory.create("sagemaker", "sagemaker-models.json", new TypeReference<Map<String, ModelResource>>() {}),+ storageFactory.create("sagemaker", "sagemaker-endpoint-configs.json", new TypeReference<Map<String, EndpointConfigResource>>() {}),+ storageFactory.create("sagemaker", "sagemaker-endpoints.json", new TypeReference<Map<String, EndpointResource>>() {}),+ storageFactory.create("sagemaker", "sagemaker-training-jobs.json", new TypeReference<Map<String, TrainingJobResource>>() {}),+ regionResolver, mapper, endpointManager, trainingRunner, Clock.systemUTC());+ }++ SageMakerService(StorageBackend<String, ModelResource> modelStore,+ StorageBackend<String, EndpointConfigResource> endpointConfigStore,+ StorageBackend<String, EndpointResource> endpointStore,+ StorageBackend<String, TrainingJobResource> trainingJobStore,+ RegionResolver regionResolver,+ ObjectMapper mapper,+ SageMakerEndpointManager endpointManager,+ SageMakerTrainingRunner trainingRunner) {+ this(modelStore, endpointConfigStore, endpointStore, trainingJobStore, regionResolver, mapper, endpointManager,+ trainingRunner, Clock.systemUTC());+ }++ SageMakerService(StorageBackend<String, ModelResource> modelStore,+ StorageBackend<String, EndpointConfigResource> endpointConfigStore,+ StorageBackend<String, EndpointResource> endpointStore,+ StorageBackend<String, TrainingJobResource> trainingJobStore,+ RegionResolver regionResolver,+ ObjectMapper mapper,+ SageMakerEndpointManager endpointManager,+ SageMakerTrainingRunner trainingRunner,+ Clock clock) {+ this.modelStore = modelStore;+ this.endpointConfigStore = endpointConfigStore;+ this.endpointStore = endpointStore;+ this.trainingJobStore = trainingJobStore;+ this.regionResolver = regionResolver;+ this.mapper = mapper;+ this.endpointManager = endpointManager;+ this.trainingRunner = trainingRunner;+ this.clock = clock;+ }++ public synchronized ObjectNode createModel(JsonNode request, String region) {+ String name = required(request, "ModelName");+ if (model(region, name).isPresent()) {+ throw new AwsException("ValidationException", "Cannot create already existing model \"" + name + "\".", 400);+ }+ JsonNode primary = request.path("PrimaryContainer");+ JsonNode containers = request.path("Containers");+ boolean hasContainers = containers.isArray() && !containers.isEmpty() && containers.get(0).isObject();+ if (!primary.isObject() && !hasContainers) {+ throw validation("PrimaryContainer or Containers is required");+ }+ ModelResource model = new ModelResource();+ model.modelName = name;+ model.modelArn = arn(region, "model/" + name);+ model.executionRoleArn = text(request, "ExecutionRoleArn");+ model.creationTime = nowMillis();+ model.region = region;+ model.accountId = regionResolver.getAccountId();+ model.primaryContainer = primary.isObject() ? map(primary) : map(containers.get(0));+ model.containers = listMap(containers);+ model.tags = tagsFromList(request.path("Tags"));+ modelStore.put(regionKey(region, name), model);+ ObjectNode out = mapper.createObjectNode();+ out.put("ModelArn", model.modelArn);+ return out;+ }++ public ObjectNode describeModel(JsonNode request, String region) {+ String name = required(request, "ModelName");+ ModelResource m = model(region, name).orElseThrow(() -> validation("Could not find model \"" + name + "\"."));+ ObjectNode out = mapper.createObjectNode();+ out.put("ModelName", m.modelName);+ out.put("ModelArn", m.modelArn);+ out.set("PrimaryContainer", mapper.valueToTree(m.primaryContainer));+ if (!m.containers.isEmpty()) {+ out.set("Containers", mapper.valueToTree(m.containers));+ }+ if (m.executionRoleArn != null) {+ out.put("ExecutionRoleArn", m.executionRoleArn);+ }+ out.put("CreationTime", epoch(m.creationTime));+ return out;+ }++ public synchronized ObjectNode deleteModel(JsonNode request, String region) {+ String name = required(request, "ModelName");+ model(region, name).orElseThrow(() -> validation("Could not find model \"" + name + "\"."));+ modelStore.delete(regionKey(region, name));+ return mapper.createObjectNode();+ }++ public ObjectNode listModels(JsonNode request, String region) {+ List<ModelResource> models = modelStore.scan(k -> k.startsWith(region + "::")).stream()+ .sorted(Comparator.comparing(m -> m.modelName)).toList();+ String nameContains = text(request, "NameContains");+ if (nameContains != null && !nameContains.isBlank()) {+ models = models.stream().filter(m -> m.modelName.contains(nameContains)).toList();+ }+ return paginate(models, request, m -> m.modelName, (m, n) -> {+ n.put("ModelName", m.modelName);+ n.put("ModelArn", m.modelArn);+ n.put("CreationTime", epoch(m.creationTime));+ }, "Models");+ }++ public synchronized ObjectNode createEndpointConfig(JsonNode request, String region) {+ String name = required(request, "EndpointConfigName");+ if (endpointConfig(region, name).isPresent()) {+ throw new AwsException("ValidationException", "Cannot create already existing endpoint configuration \"" + name + "\".", 400);+ }+ List<Map<String, Object>> variants = listMap(request.path("ProductionVariants"));+ if (variants.isEmpty()) {+ throw validation("ProductionVariants is required");+ }+ for (Map<String, Object> v : variants) {+ String modelName = SageMakerEndpointManager.string(v.get("ModelName"));+ if (modelName == null || modelName.isBlank()) {+ throw validation("ProductionVariants.ModelName is required");+ }+ if (model(region, modelName).isEmpty()) {+ throw validation("Could not find model \"" + modelName + "\".");+ }+ }+ EndpointConfigResource cfg = new EndpointConfigResource();+ cfg.endpointConfigName = name;+ cfg.endpointConfigArn = arn(region, "endpoint-config/" + name);+ cfg.productionVariants = variants;+ cfg.creationTime = nowMillis();+ cfg.region = region;+ cfg.accountId = regionResolver.getAccountId();+ cfg.tags = tagsFromList(request.path("Tags"));+ endpointConfigStore.put(regionKey(region, name), cfg);+ ObjectNode out = mapper.createObjectNode();+ out.put("EndpointConfigArn", cfg.endpointConfigArn);+ return out;+ }++ public ObjectNode describeEndpointConfig(JsonNode request, String region) {+ String name = required(request, "EndpointConfigName");+ EndpointConfigResource cfg = endpointConfig(region, name)+ .orElseThrow(() -> validation("Could not find endpoint configuration \"" + name + "\"."));+ ObjectNode out = mapper.createObjectNode();+ out.put("EndpointConfigName", cfg.endpointConfigName);+ out.put("EndpointConfigArn", cfg.endpointConfigArn);+ out.set("ProductionVariants", mapper.valueToTree(cfg.productionVariants));+ out.put("CreationTime", epoch(cfg.creationTime));+ return out;+ }++ public synchronized ObjectNode deleteEndpointConfig(JsonNode request, String region) {+ String name = required(request, "EndpointConfigName");+ endpointConfig(region, name).orElseThrow(() -> validation("Could not find endpoint configuration \"" + name + "\"."));+ endpointConfigStore.delete(regionKey(region, name));+ return mapper.createObjectNode();+ }++ public ObjectNode listEndpointConfigs(JsonNode request, String region) {+ List<EndpointConfigResource> configs = endpointConfigStore.scan(k -> k.startsWith(region + "::")).stream()+ .sorted(Comparator.comparing(c -> c.endpointConfigName)).toList();+ return paginate(configs, request, c -> c.endpointConfigName, (c, n) -> {+ n.put("EndpointConfigName", c.endpointConfigName);+ n.put("EndpointConfigArn", c.endpointConfigArn);+ n.put("CreationTime", epoch(c.creationTime));+ }, "EndpointConfigs");+ }++ public synchronized ObjectNode createEndpoint(JsonNode request, String region) {+ String name = required(request, "EndpointName");+ if (endpoint(region, name).isPresent()) {+ throw new AwsException("ValidationException", "Cannot create already existing endpoint \"" + name + "\".", 400);+ }+ String cfgName = required(request, "EndpointConfigName");+ endpointConfig(region, cfgName).orElseThrow(() -> validation("Could not find endpoint configuration \"" + cfgName + "\"."));+ EndpointResource ep = new EndpointResource();+ ep.endpointName = name;+ ep.endpointArn = arn(region, "endpoint/" + name);+ ep.endpointConfigName = cfgName;+ ep.endpointStatus = "Creating";+ ep.creationTime = nowMillis();+ ep.lastModifiedTime = ep.creationTime;+ ep.region = region;+ ep.accountId = regionResolver.getAccountId();+ ep.tags = tagsFromList(request.path("Tags"));+ ep.generation = 1;+ endpointStore.put(regionKey(region, name), ep);+ endpointManager.startEndpointAsync(ep, this);+ ObjectNode out = mapper.createObjectNode();+ out.put("EndpointArn", ep.endpointArn);+ return out;+ }++ public synchronized ObjectNode updateEndpoint(JsonNode request, String region) {+ String name = required(request, "EndpointName");+ EndpointResource ep = endpoint(region, name).orElseThrow(() -> validation("Could not find endpoint \"" + name + "\"."));+ String cfgName = required(request, "EndpointConfigName");+ endpointConfig(region, cfgName).orElseThrow(() -> validation("Could not find endpoint configuration \"" + cfgName + "\"."));+ if (ep.containerId != null) {+ endpointManager.stopEndpoint(ep);+ }+ ep.endpointConfigName = cfgName;+ ep.endpointStatus = "Creating";+ ep.failureReason = null;+ ep.containerId = null;+ ep.lastModifiedTime = nowMillis();+ ep.generation++;+ endpointStore.put(regionKey(region, name), ep);+ endpointManager.startEndpointAsync(ep, this);+ ObjectNode out = mapper.createObjectNode();+ out.put("EndpointArn", ep.endpointArn);+ return out;+ }++ public ObjectNode describeEndpoint(JsonNode request, String region) {+ String name = required(request, "EndpointName");+ EndpointResource ep = endpoint(region, name).orElseThrow(() -> validation("Could not find endpoint \"" + name + "\"."));+ ObjectNode out = mapper.createObjectNode();+ out.put("EndpointName", ep.endpointName);+ out.put("EndpointArn", ep.endpointArn);+ out.put("EndpointConfigName", ep.endpointConfigName);+ out.put("EndpointStatus", ep.endpointStatus);+ if (ep.failureReason != null) {+ out.put("FailureReason", ep.failureReason);+ }+ out.put("CreationTime", epoch(ep.creationTime));+ out.put("LastModifiedTime", epoch(ep.lastModifiedTime));+ return out;+ }++ public synchronized ObjectNode deleteEndpoint(JsonNode request, String region) {+ String name = required(request, "EndpointName");+ EndpointResource ep = endpoint(region, name).orElseThrow(() -> validation("Could not find endpoint \"" + name + "\"."));+ ep.endpointStatus = "Deleting";+ // Bump the generation before removing the record: a start worker still in flight for+ // this endpoint checks the generation against what is (no longer) in the store and will+ // no-op instead of resurrecting the record it raced with this delete.+ ep.generation++;+ endpointStore.put(regionKey(region, name), ep);+ endpointManager.stopEndpoint(ep);+ endpointStore.delete(regionKey(region, name));+ return mapper.createObjectNode();+ }++ public ObjectNode listEndpoints(JsonNode request, String region) {+ List<EndpointResource> endpoints = endpointStore.scan(k -> k.startsWith(region + "::")).stream()+ .sorted(Comparator.comparing(e -> e.endpointName)).toList();+ return paginate(endpoints, request, e -> e.endpointName, (e, n) -> {+ n.put("EndpointName", e.endpointName);+ n.put("EndpointArn", e.endpointArn);+ n.put("EndpointStatus", e.endpointStatus);+ n.put("CreationTime", epoch(e.creationTime));+ n.put("LastModifiedTime", epoch(e.lastModifiedTime));+ }, "Endpoints");+ }++ public synchronized ObjectNode createTrainingJob(JsonNode request, String region) {+ String name = required(request, "TrainingJobName");+ if (trainingJob(region, name).isPresent()) {+ throw new AwsException("ResourceInUse", "Training job already exists: " + name, 400);+ }+ if (!request.path("AlgorithmSpecification").isObject() || text(request.path("AlgorithmSpecification"), "TrainingImage") == null) {+ throw validation("AlgorithmSpecification.TrainingImage is required");+ }+ if (!request.path("OutputDataConfig").isObject() || text(request.path("OutputDataConfig"), "S3OutputPath") == null) {+ throw validation("OutputDataConfig.S3OutputPath is required");+ }+ validateS3Uri(text(request.path("OutputDataConfig"), "S3OutputPath"), "OutputDataConfig.S3OutputPath");+ for (JsonNode channel : request.path("InputDataConfig")) {+ JsonNode s3DataSource = channel.path("DataSource").path("S3DataSource");+ if (s3DataSource.isObject()) {+ validateS3Uri(text(s3DataSource, "S3Uri"), "InputDataConfig[].DataSource.S3DataSource.S3Uri");+ }+ }+ TrainingJobResource job = new TrainingJobResource();+ job.trainingJobName = name;+ job.trainingJobArn = arn(region, "training-job/" + name);+ job.trainingJobStatus = "InProgress";+ job.secondaryStatus = "Starting";+ job.creationTime = nowMillis();+ job.region = region;+ job.accountId = regionResolver.getAccountId();+ job.algorithmSpecification = map(request.path("AlgorithmSpecification"));+ job.inputDataConfig = listMap(request.path("InputDataConfig"));+ job.outputDataConfig = map(request.path("OutputDataConfig"));+ job.resourceConfig = map(request.path("ResourceConfig"));+ job.stoppingCondition = map(request.path("StoppingCondition"));+ job.hyperParameters = stringMap(request.path("HyperParameters"));+ job.tags = tagsFromList(request.path("Tags"));+ trainingJobStore.put(regionKey(region, name), job);+ trainingRunner.runAsync(job, this);+ ObjectNode out = mapper.createObjectNode();+ out.put("TrainingJobArn", job.trainingJobArn);+ return out;+ }++ public ObjectNode describeTrainingJob(JsonNode request, String region) {+ String name = required(request, "TrainingJobName");+ TrainingJobResource job = trainingJob(region, name).orElseThrow(() -> validation("Could not find training job \"" + name + "\"."));+ ObjectNode out = mapper.valueToTree(Map.of(+ "TrainingJobName", job.trainingJobName,+ "TrainingJobArn", job.trainingJobArn,+ "TrainingJobStatus", job.trainingJobStatus,+ "SecondaryStatus", job.secondaryStatus == null ? "" : job.secondaryStatus,+ "AlgorithmSpecification", job.algorithmSpecification,+ "InputDataConfig", job.inputDataConfig,+ "OutputDataConfig", job.outputDataConfig,+ "ResourceConfig", job.resourceConfig,+ "StoppingCondition", job.stoppingCondition,+ "CreationTime", epoch(job.creationTime)+ ));+ if (job.trainingStartTime > 0) {+ out.put("TrainingStartTime", epoch(job.trainingStartTime));+ }+ if (job.trainingEndTime > 0) {+ out.put("TrainingEndTime", epoch(job.trainingEndTime));+ }+ if (job.failureReason != null) {+ out.put("FailureReason", job.failureReason);+ }+ if (job.modelArtifactsS3ModelArtifacts != null) {+ ObjectNode ma = mapper.createObjectNode();+ ma.put("S3ModelArtifacts", job.modelArtifactsS3ModelArtifacts);+ out.set("ModelArtifacts", ma);+ }+ return out;+ }++ public ObjectNode listTrainingJobs(JsonNode request, String region) {+ List<TrainingJobResource> jobs = trainingJobStore.scan(k -> k.startsWith(region + "::")).stream()+ .sorted(Comparator.comparing(j -> j.trainingJobName)).toList();+ String nameContains = text(request, "NameContains");+ if (nameContains != null && !nameContains.isBlank()) {+ jobs = jobs.stream().filter(j -> j.trainingJobName.contains(nameContains)).toList();+ }+ String statusEquals = text(request, "StatusEquals");+ if (statusEquals != null && !statusEquals.isBlank()) {+ jobs = jobs.stream().filter(j -> statusEquals.equals(j.trainingJobStatus)).toList();+ }+ return paginate(jobs, request, j -> j.trainingJobName, (j, n) -> {+ n.put("TrainingJobName", j.trainingJobName);+ n.put("TrainingJobArn", j.trainingJobArn);+ n.put("TrainingJobStatus", j.trainingJobStatus);+ n.put("CreationTime", epoch(j.creationTime));+ }, "TrainingJobSummaries");+ }++ public synchronized ObjectNode stopTrainingJob(JsonNode request, String region) {+ String name = required(request, "TrainingJobName");+ TrainingJobResource job = trainingJob(region, name).orElseThrow(() -> validation("Could not find training job \"" + name + "\"."));+ // AWS only transitions InProgress/Stopping jobs; a job already in a terminal state is+ // left as-is rather than relabeled "Stopped" out from under its real outcome.+ if ("InProgress".equals(job.trainingJobStatus) || "Stopping".equals(job.trainingJobStatus)) {+ trainingRunner.stop(job);+ job.trainingJobStatus = "Stopped";+ job.secondaryStatus = "Stopped";+ job.trainingEndTime = nowMillis();+ trainingJobStore.put(regionKey(region, name), job);+ }+ return mapper.createObjectNode();+ }++ public synchronized ObjectNode addTags(JsonNode request) {+ String arn = required(request, "ResourceArn");+ Map<String, String> tags = tagsFromList(request.path("Tags"));+ resourceTags(arn).putAll(tags);+ persistByArn(arn);+ ObjectNode out = mapper.createObjectNode();+ out.set("Tags", tagsArray(resourceTags(arn)));+ return out;+ }++ public ObjectNode listTags(JsonNode request) {+ String arn = required(request, "ResourceArn");+ ObjectNode out = mapper.createObjectNode();+ out.set("Tags", tagsArray(resourceTags(arn)));+ return out;+ }++ public synchronized ObjectNode deleteTags(JsonNode request) {+ String arn = required(request, "ResourceArn");+ Map<String, String> tags = resourceTags(arn);+ request.path("TagKeys").forEach(k -> tags.remove(k.asText()));+ persistByArn(arn);+ return mapper.createObjectNode();+ }++ Optional<ModelResource> model(String region, String name) {+ return modelStore.get(regionKey(region, name));+ }++ Optional<EndpointConfigResource> endpointConfig(String region, String name) {+ return endpointConfigStore.get(regionKey(region, name));+ }++ public Optional<EndpointResource> endpoint(String region, String name) {+ return endpointStore.get(regionKey(region, name));+ }++ Optional<TrainingJobResource> trainingJob(String region, String name) {+ return trainingJobStore.get(regionKey(region, name));+ }++ public ModelResource modelForEndpoint(EndpointResource ep) {+ EndpointConfigResource cfg = endpointConfig(ep.region, ep.endpointConfigName)+ .orElseThrow(() -> validation("Could not find endpoint configuration \"" + ep.endpointConfigName + "\"."));+ String modelName = String.valueOf(cfg.productionVariants.get(0).get("ModelName"));+ return model(ep.region, modelName).orElseThrow(() -> validation("Could not find model \"" + modelName + "\"."));+ }++ /**+ * Persists the result of an endpoint start (success or failure) unless a delete or a+ * newer start has superseded it since {@code ep} was handed to {@link SageMakerEndpointManager}.+ * Returns {@code false} when the caller should tear down whatever it just started instead+ * of publishing it, because the endpoint it belongs to is gone or has moved on.+ */+ synchronized boolean finalizeEndpointStart(EndpointResource ep) {+ EndpointResource current = endpointStore.get(regionKey(ep.region, ep.endpointName)).orElse(null);+ if (current == null || current.generation != ep.generation) {+ return false;+ }+ ep.lastModifiedTime = nowMillis();+ endpointStore.put(regionKey(ep.region, ep.endpointName), ep);+ return true;+ }++ public synchronized void updateTrainingJob(TrainingJobResource job) {+ trainingJobStore.put(regionKey(job.region, job.trainingJobName), job);+ }++ private Map<String, String> resourceTags(String arn) {+ String region = arnRegion(arn);+ String name = arn.substring(arn.lastIndexOf('/') + 1);+ if (arn.contains(":model/")) return model(region, name).orElseThrow(() -> validation("Could not find model \"" + name + "\".")).tags;+ if (arn.contains(":endpoint-config/")) return endpointConfig(region, name).orElseThrow(() -> validation("Could not find endpoint configuration \"" + name + "\".")).tags;+ if (arn.contains(":endpoint/")) return endpoint(region, name).orElseThrow(() -> validation("Could not find endpoint \"" + name + "\".")).tags;+ if (arn.contains(":training-job/")) return trainingJob(region, name).orElseThrow(() -> validation("Could not find training job \"" + name + "\".")).tags;+ throw validation("ResourceArn is invalid");+ }++ private void persistByArn(String arn) {+ String region = arnRegion(arn);+ String name = arn.substring(arn.lastIndexOf('/') + 1);+ if (arn.contains(":model/")) model(region, name).ifPresent(v -> modelStore.put(regionKey(region, name), v));+ else if (arn.contains(":endpoint-config/")) endpointConfig(region, name).ifPresent(v -> endpointConfigStore.put(regionKey(region, name), v));+ else if (arn.contains(":endpoint/")) endpoint(region, name).ifPresent(v -> endpointStore.put(regionKey(region, name), v));+ else if (arn.contains(":training-job/")) trainingJob(region, name).ifPresent(v -> trainingJobStore.put(regionKey(region, name), v));+ }++ /** ARN shape: {@code arn:aws:sagemaker:<region>:<account>:<type>/<name>}. */+ private static String arnRegion(String arn) {+ String[] parts = arn.split(":", 6);+ if (parts.length < 4 || parts[3].isBlank()) {+ throw validation("ResourceArn is invalid");+ }+ return parts[3];+ }++ private static String regionKey(String region, String name) {+ return region + "::" + name;+ }++ private String arn(String region, String resource) {+ return "arn:aws:sagemaker:" + region + ":" + regionResolver.getAccountId() + ":" + resource;+ }++ private long nowMillis() {+ return clock.millis();+ }++ static double epoch(long millis) {+ return millis / 1000.0;+ }++ static AwsException validation(String message) {+ return new AwsException("ValidationException", message, 400);+ }++ static void validateS3Uri(String uri, String field) {+ try {+ S3Uri.parse(uri);+ } catch (IllegalArgumentException e) {+ throw validation(field + ": " + e.getMessage());+ }+ }++ static String required(JsonNode node, String field) {+ String value = text(node, field);+ if (value == null || value.isBlank()) {+ throw validation(field + " is required");+ }+ return value;+ }++ static String text(JsonNode node, String field) {+ JsonNode v = node.path(field);+ return v.isMissingNode() || v.isNull() ? null : v.asText();+ }++ static Map<String, Object> map(JsonNode node) {+ if (!node.isObject()) {+ return new LinkedHashMap<>();+ }+ return STATIC_MAPPER.convertValue(node, new TypeReference<LinkedHashMap<String, Object>>() {});+ }++ static Map<String, String> stringMap(JsonNode node) {+ Map<String, String> out = new LinkedHashMap<>();+ if (node.isObject()) {+ node.fields().forEachRemaining(e -> out.put(e.getKey(), e.getValue().asText()));+ }+ return out;+ }++ static List<Map<String, Object>> listMap(JsonNode node) {+ List<Map<String, Object>> out = new ArrayList<>();+ if (node.isArray()) {+ node.forEach(n -> out.add(map(n)));+ }+ return out;+ }++ static Map<String, String> tagsFromList(JsonNode node) {+ Map<String, String> out = new LinkedHashMap<>();+ if (node.isArray()) {+ node.forEach(t -> {+ String key = text(t, "Key");+ if (key != null && !key.isBlank()) {+ out.put(key, text(t, "Value"));+ }+ });+ }+ return out;+ }++ private ArrayNode tagsArray(Map<String, String> tags) {+ ArrayNode arr = mapper.createArrayNode();+ tags.forEach((k, v) -> {+ ObjectNode n = mapper.createObjectNode();+ n.put("Key", k);+ n.put("Value", v);+ arr.add(n);+ });+ return arr;+ }++ /**+ * Applies AWS's {@code MaxResults}/{@code NextToken} contract to an already-filtered,+ * name-sorted list: {@code MaxResults} (1-100, default {@value #DEFAULT_LIMIT}) caps the+ * page, and an opaque {@code NextToken} (the last name returned) resumes strictly after it,+ * matching the sort the caller applied.+ */+ private <T> ObjectNode paginate(List<T> sorted, JsonNode request, Function<T, String> nameOf,+ BiConsumer<T, ObjectNode> render, String itemsField) {+ int max = maxResults(request);+ String nextToken = text(request, "NextToken");+ List<T> remaining = sorted;+ if (nextToken != null && !nextToken.isBlank()) {+ remaining = remaining.stream().dropWhile(v -> nameOf.apply(v).compareTo(nextToken) <= 0).toList();+ }+ List<T> page = remaining.stream().limit(max).toList();+ ArrayNode arr = mapper.createArrayNode();+ page.forEach(v -> {+ ObjectNode n = mapper.createObjectNode();+ render.accept(v, n);+ arr.add(n);+ });+ ObjectNode out = mapper.createObjectNode();+ out.set(itemsField, arr);+ if (page.size() == max && remaining.size() > max) {+ out.put("NextToken", nameOf.apply(page.get(page.size() - 1)));+ }+ return out;+ }++ private static int maxResults(JsonNode request) {+ JsonNode node = request.path("MaxResults");+ if (node.isMissingNode() || node.isNull()) {+ return DEFAULT_LIMIT;+ }+ int max = node.asInt();+ if (max < 1 || max > MAX_RESULTS_CEILING) {+ throw validation("MaxResults must be between 1 and " + MAX_RESULTS_CEILING + ".");+ }+ return max;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.javanew file mode 100644index 0000000000..de6a063964--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerTrainingRunner.java@@ -0,0 +1,362 @@+package io.github.hectorvent.floci.services.sagemaker;++import com.github.dockerjava.api.command.InspectContainerResponse;+import com.github.dockerjava.api.exception.NotFoundException;+import com.github.dockerjava.api.model.Frame;+import com.github.dockerjava.core.command.LogContainerResultCallback;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.ContainerTeardown;+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;+import org.jboss.logging.Logger;++import java.io.ByteArrayInputStream;+import java.io.ByteArrayOutputStream;+import java.io.Closeable;+import java.io.InputStream;+import java.nio.charset.StandardCharsets;+import java.nio.file.Path;+import java.time.Duration;+import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;++@ApplicationScoped+public class SageMakerTrainingRunner implements ContainerTeardown {+ private static final Logger LOG = Logger.getLogger(SageMakerTrainingRunner.class);+ private static final String LOG_GROUP = "/aws/sagemaker/TrainingJobs";++ private final ContainerBuilder containerBuilder;+ private final ContainerLifecycleManager lifecycleManager;+ private final ContainerLogStreamer logStreamer;+ private final EmulatorConfig config;+ private final ContainerDetector containerDetector;+ private final S3Service s3Service;+ private final ObjectMapper mapper;+ private final ExecutorService executor = Executors.newCachedThreadPool();+ private final ConcurrentHashMap<String, String> containers = new ConcurrentHashMap<>();+ // Names a stop that stop() has already committed to the store as "Stopped": run()'s exit-code+ // handling checks this before persisting, since a stop races the removal of the very container+ // waitForExit() is polling, and the container disappearing (NotFoundException, exit code 1)+ // must not be relabeled a training failure.+ private final Set<String> stopRequested = ConcurrentHashMap.newKeySet();++ @Inject+ public SageMakerTrainingRunner(ContainerBuilder containerBuilder, ContainerLifecycleManager lifecycleManager,+ ContainerLogStreamer logStreamer, EmulatorConfig config,+ ContainerDetector containerDetector, S3Service s3Service, ObjectMapper mapper) {+ this.containerBuilder = containerBuilder;+ this.lifecycleManager = lifecycleManager;+ this.logStreamer = logStreamer;+ this.config = config;+ this.containerDetector = containerDetector;+ this.s3Service = s3Service;+ this.mapper = mapper;+ }++ public void runAsync(TrainingJobResource job, SageMakerService service) {+ executor.submit(() -> run(job, service));+ }++ private void run(TrainingJobResource job, SageMakerService service) {+ String containerId = null;+ Closeable logs = null;+ try {+ String image = SageMakerEndpointManager.string(job.algorithmSpecification.get("TrainingImage"));+ String name = ContainerStorageHelper.dockerName(config, "floci-sagemaker-training-" + job.trainingJobName);+ lifecycleManager.removeIfExists(name);+ ContainerBuilder.Builder builder = containerBuilder.newContainer(image)+ .withName(name)+ .withEnv(environment(job))+ .withDockerNetwork(config.services().sagemaker().dockerNetwork())+ .withHostDockerInternalOnLinux()+ .withEmbeddedDns()+ .withLogRotation();+ SageMakerEndpointManager.applyEntrypoint(builder, job.algorithmSpecification, "train");+ ContainerSpec spec = builder.build();+ containerId = lifecycleManager.create(spec);+ containers.put(job.trainingJobName, containerId);+ job.containerId = containerId;+ copyTrainingInput(containerId, job);+ lifecycleManager.startCreated(containerId, spec);+ job.trainingStartTime = System.currentTimeMillis();+ job.secondaryStatus = "Training";+ service.updateTrainingJob(job);+ logs = logStreamer.attach(containerId, LOG_GROUP, job.trainingJobName + "/algo-1", job.region,+ "sagemaker:" + job.trainingJobName);+ Integer exit = waitForExit(containerId, timeout(job));+ if (stopRequested.remove(job.trainingJobName)) {+ // stop() already persisted Stopped and is tearing the container down itself;+ // whatever exit code we just observed (or failed to) is a side effect of that,+ // not a real outcome to report.+ containers.remove(job.trainingJobName);+ return;+ }+ if (exit == null) {+ lifecycleManager.stopAndRemove(containerId, logs);+ job.trainingJobStatus = "Failed";+ job.secondaryStatus = "Failed";+ job.failureReason = "Training job exceeded StoppingCondition.MaxRuntimeInSeconds";+ } else if (exit == 0) {+ job.modelArtifactsS3ModelArtifacts = uploadModelArtifacts(containerId, job);+ lifecycleManager.stopAndRemove(containerId, logs);+ job.trainingJobStatus = "Completed";+ job.secondaryStatus = "Completed";+ } else {+ job.failureReason = readFailure(containerId, exit);+ lifecycleManager.stopAndRemove(containerId, logs);+ job.trainingJobStatus = "Failed";+ job.secondaryStatus = "Failed";+ }+ containers.remove(job.trainingJobName);+ job.trainingEndTime = System.currentTimeMillis();+ service.updateTrainingJob(job);+ } catch (Exception e) {+ if (stopRequested.remove(job.trainingJobName)) {+ containers.remove(job.trainingJobName);+ return;+ }+ LOG.warnv("SageMaker training job {0} failed: {1}", job.trainingJobName, e.getMessage());+ if (containerId != null) {+ lifecycleManager.stopAndRemove(containerId, logs);+ }+ containers.remove(job.trainingJobName);+ job.trainingJobStatus = "Failed";+ job.secondaryStatus = "Failed";+ job.failureReason = e.getMessage();+ job.trainingEndTime = System.currentTimeMillis();+ service.updateTrainingJob(job);+ }+ }++ public void stop(TrainingJobResource job) {+ stopRequested.add(job.trainingJobName);+ String id = job.containerId != null ? job.containerId : containers.remove(job.trainingJobName);+ if (id != null) {+ lifecycleManager.stopAndRemove(id, null);+ }+ }++ @Override+ public void stopManagedContainers() {+ containers.forEach((name, id) -> lifecycleManager.stopAndRemove(id, null));+ containers.clear();+ executor.shutdownNow();+ }++ private List<String> environment(TrainingJobResource job) {+ List<String> env = new ArrayList<>();+ env.add("TRAINING_JOB_NAME=" + job.trainingJobName);+ env.add("TRAINING_JOB_ARN=" + job.trainingJobArn);+ env.add("SAGEMAKER_REGION=" + job.region);+ env.add("AWS_REGION=" + job.region);+ env.add("AWS_DEFAULT_REGION=" + job.region);+ env.add("AWS_ACCESS_KEY_ID=test");+ env.add("AWS_SECRET_ACCESS_KEY=test");+ env.add("AWS_SESSION_TOKEN=test");+ String endpoint = "http://" + resolveEndpointHostname() + ":" + config.port();+ env.add("AWS_ENDPOINT_URL=" + endpoint);+ env.add("FLOCI_ENDPOINT=" + endpoint);+ return env;+ }++ private String resolveEndpointHostname() {+ return containerDetector.isRunningInContainer() ? config.hostname().orElse(EmbeddedDnsServer.DEFAULT_SUFFIX) : "host.docker.internal";+ }++ private void copyTrainingInput(String containerId, TrainingJobResource job) throws Exception {+ ByteArrayOutputStream out = new ByteArrayOutputStream();+ try (TarArchiveOutputStream tar = new TarArchiveOutputStream(out)) {+ SageMakerEndpointManager.addFile(tar, "opt/ml/input/config/hyperparameters.json",+ mapper.writeValueAsBytes(job.hyperParameters));+ SageMakerEndpointManager.addFile(tar, "opt/ml/input/config/resourceconfig.json",+ "{\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\"]}".getBytes(StandardCharsets.UTF_8));+ SageMakerEndpointManager.addFile(tar, "opt/ml/input/config/inputdataconfig.json",+ mapper.writeValueAsBytes(inputDataConfig(job.inputDataConfig)));+ for (Map<String, Object> channel : job.inputDataConfig) {+ copyChannelObjects(tar, channel);+ }+ }+ lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)+ .withRemotePath("/")+ .withTarInputStream(new ByteArrayInputStream(out.toByteArray()))+ .exec();+ }++ private Map<String, Object> inputDataConfig(List<Map<String, Object>> channels) {+ Map<String, Object> out = new LinkedHashMap<>();+ for (Map<String, Object> c : channels) {+ Map<String, Object> value = new LinkedHashMap<>();+ value.put("ContentType", c.getOrDefault("ContentType", "application/octet-stream"));+ value.put("TrainingInputMode", c.getOrDefault("TrainingInputMode", "File"));+ out.put(String.valueOf(c.get("ChannelName")), value);+ }+ return out;+ }++ @SuppressWarnings("unchecked")+ private void copyChannelObjects(TarArchiveOutputStream tar, Map<String, Object> channel) throws Exception {+ String channelName = String.valueOf(channel.get("ChannelName"));+ Object dataSource = channel.get("DataSource");+ if (!(dataSource instanceof Map<?, ?> ds)) {+ return;+ }+ Object s3 = ds.get("S3DataSource");+ if (!(s3 instanceof Map<?, ?> s3ds)) {+ return;+ }+ String s3Uri = String.valueOf(s3ds.get("S3Uri"));+ S3Uri uri = S3Uri.parse(s3Uri);+ List<S3Object> objects = uri.key().isBlank()+ ? s3Service.listObjects(uri.bucket(), "", null, 1000)+ : s3Service.listObjects(uri.bucket(), uri.key(), null, 1000);+ for (S3Object object : objects) {+ S3Object full = s3Service.getObject(uri.bucket(), object.getKey());+ String relative = relativeToPrefix(uri.key(), object.getKey());+ String safeRelative = safeRelativePath(relative);+ if (safeRelative != null && !safeRelative.isBlank()) {+ SageMakerEndpointManager.addFile(tar, "opt/ml/input/data/" + channelName + "/" + safeRelative, full.getData());+ }+ }+ }++ /**+ * The part of an object key below the channel's S3 prefix, preserving any subdirectory+ * structure instead of flattening every object to its basename (which loses structure and+ * collides same-named objects in different "folders" under the prefix).+ */+ private static String relativeToPrefix(String prefix, String key) {+ if (prefix.isBlank()) {+ return key;+ }+ String folder = prefix.endsWith("/") ? prefix : prefix + "/";+ if (key.startsWith(folder)) {+ return key.substring(folder.length());+ }+ // The prefix matched key without a folder boundary (e.g. prefix "input" matching+ // "input.csv" itself, or a same-named sibling key equal to the prefix): fall back to+ // the basename rather than a key that still carries the shared prefix.+ return key.equals(prefix) || key.startsWith(prefix)+ ? key.substring(key.lastIndexOf('/') + 1)+ : key;+ }++ /** Rejects an absolute path or one that escapes the channel directory via "..". */+ private static String safeRelativePath(String relative) {+ Path normalized = Path.of(relative).normalize();+ if (normalized.isAbsolute() || normalized.startsWith("..") || normalized.toString().equals("..")) {+ return null;+ }+ return normalized.toString();+ }++ private Integer waitForExit(String containerId, Duration timeout) throws InterruptedException {+ long deadline = timeout.isZero() ? Long.MAX_VALUE : System.currentTimeMillis() + timeout.toMillis();+ while (System.currentTimeMillis() < deadline) {+ Integer exit = getExitCodeIfStopped(containerId);+ if (exit != null) {+ return exit;+ }+ Thread.sleep(500);+ }+ return null;+ }++ private Integer getExitCodeIfStopped(String containerId) {+ try {+ InspectContainerResponse inspect = lifecycleManager.getDockerClient().inspectContainerCmd(containerId).exec();+ if (Boolean.TRUE.equals(inspect.getState().getRunning())) {+ return null;+ }+ Long exit = inspect.getState().getExitCodeLong();+ return exit == null ? 0 : exit.intValue();+ } catch (NotFoundException e) {+ return 1;+ }+ }++ private Duration timeout(TrainingJobResource job) {+ Object value = job.stoppingCondition.get("MaxRuntimeInSeconds");+ if (value instanceof Number n) {+ return Duration.ofSeconds(n.longValue());+ }+ return Duration.ZERO;+ }++ private String uploadModelArtifacts(String containerId, TrainingJobResource job) throws Exception {+ byte[] data = modelTarGz(containerId);+ S3Uri out = S3Uri.parse(String.valueOf(job.outputDataConfig.get("S3OutputPath")));+ String prefix = out.key().isBlank() ? "" : (out.key().endsWith("/") ? out.key() : out.key() + "/");+ String key = prefix + job.trainingJobName + "/output/model.tar.gz";+ s3Service.putObject(out.bucket(), key, data, "application/x-tar", Map.of());+ return "s3://" + out.bucket() + "/" + key;+ }++ private byte[] modelTarGz(String containerId) throws Exception {+ ByteArrayOutputStream out = new ByteArrayOutputStream();+ try (GzipCompressorOutputStream gz = new GzipCompressorOutputStream(out);+ TarArchiveOutputStream tarOut = new TarArchiveOutputStream(gz);+ InputStream dockerTar = lifecycleManager.getDockerClient().copyArchiveFromContainerCmd(containerId, "/opt/ml/model").exec();+ TarArchiveInputStream tarIn = new TarArchiveInputStream(dockerTar)) {+ TarArchiveEntry e;+ String prefix = null;+ while ((e = tarIn.getNextEntry()) != null) {+ if (!tarIn.canReadEntryData(e) || e.isDirectory()) continue;+ String name = e.getName();+ if (prefix == null) {+ int slash = name.indexOf('/');+ prefix = slash >= 0 ? name.substring(0, slash + 1) : "";+ }+ if (!prefix.isBlank() && name.startsWith(prefix)) {+ name = name.substring(prefix.length());+ }+ if (!name.isBlank()) {+ SageMakerEndpointManager.addFile(tarOut, name, tarIn.readAllBytes());+ }+ }+ }+ return out.toByteArray();+ }++ private String readFailure(String containerId, int exitCode) {+ StringBuilder sb = new StringBuilder();+ try {+ lifecycleManager.getDockerClient().logContainerCmd(containerId)+ .withStdErr(true).withTail(20)+ .exec(new LogContainerResultCallback() {+ @Override+ public void onNext(Frame item) {+ sb.append(new String(item.getPayload(), StandardCharsets.UTF_8));+ }+ }).awaitCompletion(Duration.ofSeconds(2).toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS);+ } catch (InterruptedException e) {+ LOG.warnv("Interrupted while reading SageMaker training failure log for container {0}: {1}", containerId, e.getMessage());+ Thread.currentThread().interrupt();+ return "Container exited with code " + exitCode;+ } catch (Exception e) {+ LOG.warnv("Could not read SageMaker training failure log for container {0}: {1}", containerId, e.getMessage());+ }+ String detail = sb.toString().trim();+ return detail.isBlank() ? "Container exited with code " + exitCode : detail;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/sagemaker/model/SageMakerEntities.java b/src/main/java/io/github/hectorvent/floci/services/sagemaker/model/SageMakerEntities.javanew file mode 100644index 0000000000..57482a3ad4--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/sagemaker/model/SageMakerEntities.java@@ -0,0 +1,88 @@+package io.github.hectorvent.floci.services.sagemaker.model;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++@RegisterForReflection+public final class SageMakerEntities {+ private SageMakerEntities() {+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ public static class ModelResource {+ public String modelName;+ public String modelArn;+ public Map<String, Object> primaryContainer = new LinkedHashMap<>();+ public List<Map<String, Object>> containers = new ArrayList<>();+ public String executionRoleArn;+ public long creationTime;+ public String region;+ public String accountId;+ public Map<String, String> tags = new LinkedHashMap<>();+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ public static class EndpointConfigResource {+ public String endpointConfigName;+ public String endpointConfigArn;+ public List<Map<String, Object>> productionVariants = new ArrayList<>();+ public long creationTime;+ public String region;+ public String accountId;+ public Map<String, String> tags = new LinkedHashMap<>();+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ public static class EndpointResource {+ public String endpointName;+ public String endpointArn;+ public String endpointConfigName;+ public String endpointStatus;+ public String failureReason;+ public long creationTime;+ public long lastModifiedTime;+ public String region;+ public String accountId;+ public String containerId;+ public String invokeHost;+ public int invokePort;+ // Bumped every time a start (create or update) is kicked off; a start worker only+ // persists its result while it is still the current one for this endpoint, so a+ // superseding update or a delete does not get overwritten by a stale worker finishing+ // late.+ public long generation;+ public Map<String, String> tags = new LinkedHashMap<>();+ }++ @RegisterForReflection+ @JsonIgnoreProperties(ignoreUnknown = true)+ public static class TrainingJobResource {+ public String trainingJobName;+ public String trainingJobArn;+ public String trainingJobStatus;+ public String secondaryStatus;+ public String failureReason;+ public long creationTime;+ public long trainingStartTime;+ public long trainingEndTime;+ public String region;+ public String accountId;+ public Map<String, Object> algorithmSpecification = new LinkedHashMap<>();+ public List<Map<String, Object>> inputDataConfig = new ArrayList<>();+ public Map<String, Object> outputDataConfig = new LinkedHashMap<>();+ public Map<String, Object> resourceConfig = new LinkedHashMap<>();+ public Map<String, Object> stoppingCondition = new LinkedHashMap<>();+ public Map<String, String> hyperParameters = new LinkedHashMap<>();+ public String modelArtifactsS3ModelArtifacts;+ public String containerId;+ public Map<String, String> tags = new LinkedHashMap<>();+ }+}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex d738af19bf..3f272adce3 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -165,6 +165,8 @@ floci:flush-interval-ms: 5000opensearch:flush-interval-ms: 5000+ sagemaker:+ flush-interval-ms: 5000batch:flush-interval-ms: 5000lightsail:@@ -600,6 +602,9 @@ floci:codebuild:enabled: true# docker-network: floci-network+ sagemaker:+ enabled: true+ # docker-network: floci-networkbatch:enabled: truerunner-mode: immediatediff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.javanew file mode 100644index 0000000000..b9ff351f87--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java@@ -0,0 +1,182 @@+package io.github.hectorvent.floci.services.sagemaker;++import com.github.dockerjava.api.DockerClient;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;+import org.junit.jupiter.api.Assumptions;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.nio.charset.StandardCharsets;+import java.time.Duration;+import java.util.Map;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.notNullValue;+import static org.junit.jupiter.api.Assertions.assertNotNull;++@QuarkusTest+class SageMakerDockerIntegrationTest {+ private static final String AUTH =+ "AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/sagemaker/aws4_request";++ @Inject DockerClient dockerClient;+ @Inject S3Service s3Service;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @BeforeEach+ void requireDocker() {+ Assumptions.assumeTrue(isDockerAvailable(), "Docker daemon must be available for SageMaker Docker tests");+ }++ @Test+ void trainingJobRunsRealContainerAndUploadsModelArtifacts() throws Exception {+ String suffix = Long.toString(System.nanoTime(), 36);+ String bucket = "sm-train-" + suffix;+ s3Service.createBucket(bucket, "us-east-1");+ s3Service.putObject(bucket, "input/data.txt", "hello".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());+ String job = "sm-train-" + suffix;+ post("SageMaker.CreateTrainingJob", """+ {+ "TrainingJobName":"%s",+ "AlgorithmSpecification":{+ "TrainingImage":"public.ecr.aws/docker/library/busybox:stable",+ "ContainerEntrypoint":["/bin/sh","-c"],+ "ContainerArguments":["mkdir -p /opt/ml/model && cp /opt/ml/input/data/train/data.txt /opt/ml/model/model.txt"]+ },+ "InputDataConfig":[{"ChannelName":"train","DataSource":{"S3DataSource":{"S3Uri":"s3://%s/input"}},"ContentType":"text/plain","TrainingInputMode":"File"}],+ "OutputDataConfig":{"S3OutputPath":"s3://%s/output"},+ "ResourceConfig":{"InstanceType":"ml.m5.large","InstanceCount":1,"VolumeSizeInGB":1},+ "StoppingCondition":{"MaxRuntimeInSeconds":60}+ }+ """.formatted(job, bucket, bucket)).then().statusCode(200).body("TrainingJobArn", notNullValue());++ waitForTraining(job, "Completed");+ String artifact = post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job))+ .then().statusCode(200).body("TrainingJobStatus", equalTo("Completed"))+ .extract().path("ModelArtifacts.S3ModelArtifacts");+ S3Uri uri = S3Uri.parse(artifact);+ assertNotNull(s3Service.getObject(uri.bucket(), uri.key()).getData());+ }++ @Test+ void endpointHostsRealContainerAndRuntimeProxiesInvocations() throws Exception {+ String suffix = Long.toString(System.nanoTime(), 36);+ String model = "sm-ep-model-" + suffix;+ String cfg = "sm-ep-cfg-" + suffix;+ String endpoint = "sm-endpoint-" + suffix;+ String script = "cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200 if self.path == '/ping' else 404); self.end_headers()\n def do_POST(self):\n n=int(self.headers.get('content-length','0')); b=self.rfile.read(n); self.send_response(200); self.send_header('Content-Type','text/plain'); self.end_headers(); self.wfile.write(b.upper())\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py";+ post("SageMaker.CreateModel", """+ {"ModelName":"%s","PrimaryContainer":{"Image":"public.ecr.aws/docker/library/python:3-alpine","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}+ """.formatted(model, json(script))).then().statusCode(200);+ post("SageMaker.CreateEndpointConfig", """+ {"EndpointConfigName":"%s","ProductionVariants":[{"VariantName":"AllTraffic","ModelName":"%s","InitialInstanceCount":1,"InstanceType":"ml.t2.medium","InitialVariantWeight":1.0}]}+ """.formatted(cfg, model)).then().statusCode(200);+ post("SageMaker.CreateEndpoint", "{\"EndpointName\":\"%s\",\"EndpointConfigName\":\"%s\"}".formatted(endpoint, cfg))+ .then().statusCode(200);+ waitForEndpoint(endpoint, "InService");+ given().header("Authorization", "AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/runtime.sagemaker/aws4_request")+ .contentType("text/plain").body("hello")+ .when().post("/endpoints/%s/invocations".formatted(endpoint))+ .then().statusCode(200).body(equalTo("HELLO"));+ post("SageMaker.DeleteEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(200);+ }++ @Test+ void stoppingATrainingJobDoesNotGetRelabeledFailed() throws Exception {+ String suffix = Long.toString(System.nanoTime(), 36);+ String bucket = "sm-stop-" + suffix;+ s3Service.createBucket(bucket, "us-east-1");+ String job = "sm-stop-" + suffix;+ post("SageMaker.CreateTrainingJob", """+ {+ "TrainingJobName":"%s",+ "AlgorithmSpecification":{+ "TrainingImage":"public.ecr.aws/docker/library/busybox:stable",+ "ContainerEntrypoint":["/bin/sh","-c"],+ "ContainerArguments":["sleep 60"]+ },+ "OutputDataConfig":{"S3OutputPath":"s3://%s/output"},+ "ResourceConfig":{"InstanceType":"ml.m5.large","InstanceCount":1,"VolumeSizeInGB":1},+ "StoppingCondition":{"MaxRuntimeInSeconds":300}+ }+ """.formatted(job, bucket)).then().statusCode(200);+ // Give the container time to actually be created and start running "sleep 60" so the+ // stop races a real running container rather than one still being staged.+ Thread.sleep(3000);+ post("SageMaker.StopTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job)).then().statusCode(200);+ post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job))+ .then().statusCode(200).body("TrainingJobStatus", equalTo("Stopped"));+ // The runner's exit-code handling races the container disappearing after the stop;+ // give it a beat and confirm it did not relabel the stop a failure once it observes that.+ Thread.sleep(3000);+ post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job))+ .then().statusCode(200).body("TrainingJobStatus", equalTo("Stopped"));+ }++ @Test+ void deletingAnEndpointDuringStartupDoesNotResurrectIt() throws Exception {+ String suffix = Long.toString(System.nanoTime(), 36);+ String model = "sm-del-model-" + suffix;+ String cfg = "sm-del-cfg-" + suffix;+ String endpoint = "sm-del-endpoint-" + suffix;+ // Sleeps before serving so the /ping health check keeps CreateEndpoint's async start+ // in flight well past DeleteEndpoint returning.+ String script = "sleep 5 && cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200); self.end_headers()\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py";+ post("SageMaker.CreateModel", """+ {"ModelName":"%s","PrimaryContainer":{"Image":"public.ecr.aws/docker/library/python:3-alpine","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}+ """.formatted(model, json(script))).then().statusCode(200);+ post("SageMaker.CreateEndpointConfig", """+ {"EndpointConfigName":"%s","ProductionVariants":[{"VariantName":"AllTraffic","ModelName":"%s","InitialInstanceCount":1,"InstanceType":"ml.t2.medium","InitialVariantWeight":1.0}]}+ """.formatted(cfg, model)).then().statusCode(200);+ post("SageMaker.CreateEndpoint", "{\"EndpointName\":\"%s\",\"EndpointConfigName\":\"%s\"}".formatted(endpoint, cfg))+ .then().statusCode(200);+ post("SageMaker.DeleteEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(200);+ // Let the superseded start worker finish (it should discard its result, not persist it).+ Thread.sleep(10000);+ post("SageMaker.DescribeEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(400);+ }++ private void waitForTraining(String job, String status) throws Exception {+ long deadline = System.currentTimeMillis() + Duration.ofSeconds(120).toMillis();+ while (System.currentTimeMillis() < deadline) {+ String current = post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job)).then().statusCode(200).extract().path("TrainingJobStatus");+ if (status.equals(current)) return;+ if ("Failed".equals(current)) throw new AssertionError("Training failed");+ Thread.sleep(1000);+ }+ throw new AssertionError("Timed out waiting for training job");+ }++ private void waitForEndpoint(String endpoint, String status) throws Exception {+ long deadline = System.currentTimeMillis() + Duration.ofSeconds(150).toMillis();+ while (System.currentTimeMillis() < deadline) {+ String current = post("SageMaker.DescribeEndpoint", "{\"EndpointName\":\"%s\"}".formatted(endpoint)).then().statusCode(200).extract().path("EndpointStatus");+ if (status.equals(current)) return;+ if ("Failed".equals(current)) throw new AssertionError("Endpoint failed");+ Thread.sleep(1000);+ }+ throw new AssertionError("Timed out waiting for endpoint");+ }++ private io.restassured.response.Response post(String target, String body) {+ return given().header("Authorization", AUTH).header("X-Amz-Target", target)+ .contentType("application/x-amz-json-1.1").body(body).when().post("/");+ }++ private boolean isDockerAvailable() {+ try { dockerClient.pingCmd().exec(); return true; } catch (Exception e) { return false; }+ }++ private String json(String value) {+ return "\"" + value.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n") + "\"";+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerIntegrationTest.javanew file mode 100644index 0000000000..a2a2189695--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerIntegrationTest.java@@ -0,0 +1,67 @@+package io.github.hectorvent.floci.services.sagemaker;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.hasSize;+import static org.hamcrest.Matchers.notNullValue;++@QuarkusTest+class SageMakerIntegrationTest {+ private static final String AUTH =+ "AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/sagemaker/aws4_request";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void jsonControlPlaneModelEndpointConfigAndTags() {+ String suffix = Long.toString(System.nanoTime(), 36);+ String model = "sm-model-" + suffix;+ String arn = post("SageMaker.CreateModel", """+ {"ModelName":"%s","PrimaryContainer":{"Image":"busybox:stable"},"Tags":[{"Key":"env","Value":"test"}]}+ """.formatted(model))+ .then().statusCode(200).body("ModelArn", notNullValue()).extract().path("ModelArn");++ post("SageMaker.DescribeModel", "{\"ModelName\":\"%s\"}".formatted(model))+ .then().statusCode(200).body("ModelName", equalTo(model)).body("PrimaryContainer.Image", equalTo("busybox:stable"));++ String cfg = "sm-cfg-" + suffix;+ post("SageMaker.CreateEndpointConfig", """+ {"EndpointConfigName":"%s","ProductionVariants":[{"VariantName":"AllTraffic","ModelName":"%s","InitialInstanceCount":1,"InstanceType":"ml.t2.medium","InitialVariantWeight":1.0}]}+ """.formatted(cfg, model))+ .then().statusCode(200).body("EndpointConfigArn", notNullValue());+ post("SageMaker.ListEndpointConfigs", "{}").then().statusCode(200)+ .body("EndpointConfigs.find { it.EndpointConfigName == '%s' }.EndpointConfigName".formatted(cfg), equalTo(cfg));++ post("SageMaker.AddTags", "{\"ResourceArn\":\"%s\",\"Tags\":[{\"Key\":\"owner\",\"Value\":\"sdk\"}]}".formatted(arn))+ .then().statusCode(200).body("Tags", hasSize(2));+ post("SageMaker.DeleteTags", "{\"ResourceArn\":\"%s\",\"TagKeys\":[\"env\"]}".formatted(arn))+ .then().statusCode(200);+ post("SageMaker.ListTags", "{\"ResourceArn\":\"%s\"}".formatted(arn))+ .then().statusCode(200).body("Tags", hasSize(1)).body("Tags[0].Key", equalTo("owner"));++ post("SageMaker.DeleteEndpointConfig", "{\"EndpointConfigName\":\"%s\"}".formatted(cfg)).then().statusCode(200);+ post("SageMaker.DeleteModel", "{\"ModelName\":\"%s\"}".formatted(model)).then().statusCode(200);+ }++ @Test+ void trainingJobValidationErrorsUseAwsJsonErrorShape() {+ post("SageMaker.CreateTrainingJob", "{\"TrainingJobName\":\"bad\"}")+ .then().statusCode(400).body("__type", equalTo("ValidationException"));+ }++ private io.restassured.response.Response post(String target, String body) {+ return given().header("Authorization", AUTH)+ .header("X-Amz-Target", target)+ .contentType("application/x-amz-json-1.1")+ .body(body)+ .when().post("/");+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerServiceTest.javanew file mode 100644index 0000000000..4b5aae4a1c--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerServiceTest.java@@ -0,0 +1,171 @@+package io.github.hectorvent.floci.services.sagemaker;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointConfigResource;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.ModelResource;+import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;+import org.junit.jupiter.api.Test;++import java.util.List;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.mock;++class SageMakerServiceTest {+ private final ObjectMapper mapper = new ObjectMapper();++ @Test+ void modelCrudAndDuplicateValidation() throws Exception {+ SageMakerService service = service();+ service.createModel(mapper.readTree("""+ {"ModelName":"m1","PrimaryContainer":{"Image":"busybox:stable"},"ExecutionRoleArn":"arn:aws:iam::000000000000:role/r"}+ """), "us-east-1");+ assertEquals("m1", service.describeModel(mapper.readTree("{\"ModelName\":\"m1\"}"), "us-east-1").path("ModelName").asText());+ AwsException duplicate = assertThrows(AwsException.class, () -> service.createModel(mapper.readTree("""+ {"ModelName":"m1","PrimaryContainer":{"Image":"busybox:stable"}}+ """), "us-east-1"));+ assertEquals("ValidationException", duplicate.getErrorCode());+ service.deleteModel(mapper.readTree("{\"ModelName\":\"m1\"}"), "us-east-1");+ AwsException missing = assertThrows(AwsException.class,+ () -> service.describeModel(mapper.readTree("{\"ModelName\":\"m1\"}"), "us-east-1"));+ assertEquals("ValidationException", missing.getErrorCode());+ AwsException deleteMissing = assertThrows(AwsException.class,+ () -> service.deleteModel(mapper.readTree("{\"ModelName\":\"m1\"}"), "us-east-1"));+ assertEquals("ValidationException", deleteMissing.getErrorCode());+ }++ @Test+ void modelIsScopedByRegion() throws Exception {+ SageMakerService service = service();+ service.createModel(mapper.readTree("""+ {"ModelName":"m1","PrimaryContainer":{"Image":"busybox:stable"}}+ """), "us-east-1");+ AwsException missingInOtherRegion = assertThrows(AwsException.class,+ () -> service.describeModel(mapper.readTree("{\"ModelName\":\"m1\"}"), "us-west-2"));+ assertEquals("ValidationException", missingInOtherRegion.getErrorCode());+ // Same name, different region: not a collision, since the ARNs are regional.+ service.createModel(mapper.readTree("""+ {"ModelName":"m1","PrimaryContainer":{"Image":"busybox:stable"}}+ """), "us-west-2");+ assertEquals(1, service.listModels(mapper.readTree("{}"), "us-east-1").path("Models").size());+ assertEquals(1, service.listModels(mapper.readTree("{}"), "us-west-2").path("Models").size());+ }++ @Test+ void createModelRejectsEmptyContainers() throws Exception {+ SageMakerService service = service();+ AwsException empty = assertThrows(AwsException.class, () -> service.createModel(mapper.readTree("""+ {"ModelName":"m1","Containers":[]}+ """), "us-east-1"));+ assertEquals("ValidationException", empty.getErrorCode());+ }++ @Test+ void listModelsValidatesMaxResultsAndPaginates() throws Exception {+ SageMakerService service = service();+ for (String name : List.of("a", "b", "c")) {+ service.createModel(mapper.readTree(+ "{\"ModelName\":\"" + name + "\",\"PrimaryContainer\":{\"Image\":\"busybox:stable\"}}"), "us-east-1");+ }+ AwsException badMax = assertThrows(AwsException.class,+ () -> service.listModels(mapper.readTree("{\"MaxResults\":0}"), "us-east-1"));+ assertEquals("ValidationException", badMax.getErrorCode());++ JsonNode firstPage = service.listModels(mapper.readTree("{\"MaxResults\":2}"), "us-east-1");+ assertEquals(2, firstPage.path("Models").size());+ assertEquals("a", firstPage.path("Models").get(0).path("ModelName").asText());+ String nextToken = firstPage.path("NextToken").asText();+ assertEquals("b", nextToken);++ JsonNode secondPage = service.listModels(+ mapper.readTree("{\"MaxResults\":2,\"NextToken\":\"" + nextToken + "\"}"), "us-east-1");+ assertEquals(1, secondPage.path("Models").size());+ assertEquals("c", secondPage.path("Models").get(0).path("ModelName").asText());+ assertTrue(secondPage.path("NextToken").isMissingNode());+ }++ @Test+ void deleteEndpointConfigRequiresExistingConfig() throws Exception {+ SageMakerService service = service();+ AwsException missing = assertThrows(AwsException.class,+ () -> service.deleteEndpointConfig(mapper.readTree("{\"EndpointConfigName\":\"missing\"}"), "us-east-1"));+ assertEquals("ValidationException", missing.getErrorCode());+ }++ @Test+ void endpointConfigRequiresExistingModel() throws Exception {+ SageMakerService service = service();+ AwsException missing = assertThrows(AwsException.class, () -> service.createEndpointConfig(mapper.readTree("""+ {"EndpointConfigName":"cfg","ProductionVariants":[{"VariantName":"AllTraffic","ModelName":"missing","InitialInstanceCount":1,"InstanceType":"ml.t2.medium"}]}+ """), "us-east-1"));+ assertEquals("ValidationException", missing.getErrorCode());+ }++ @Test+ void tagsRoundTrip() throws Exception {+ SageMakerService service = service();+ String arn = service.createModel(mapper.readTree("""+ {"ModelName":"tagged","PrimaryContainer":{"Image":"busybox:stable"},"Tags":[{"Key":"a","Value":"b"}]}+ """), "us-east-1").path("ModelArn").asText();+ service.addTags(mapper.readTree("{\"ResourceArn\":\"" + arn + "\",\"Tags\":[{\"Key\":\"c\",\"Value\":\"d\"}]}"));+ assertEquals(2, service.listTags(mapper.readTree("{\"ResourceArn\":\"" + arn + "\"}")).path("Tags").size());+ service.deleteTags(mapper.readTree("{\"ResourceArn\":\"" + arn + "\",\"TagKeys\":[\"a\"]}"));+ assertEquals("c", service.listTags(mapper.readTree("{\"ResourceArn\":\"" + arn + "\"}")).path("Tags").get(0).path("Key").asText());+ }++ @Test+ void tagsWithoutAKeyAreSkippedInsteadOfStoredWithANullKey() throws Exception {+ SageMakerService service = service();+ String arn = service.createModel(mapper.readTree("""+ {"ModelName":"m1","PrimaryContainer":{"Image":"busybox:stable"},"Tags":[{"Value":"orphan"},{"Key":"a","Value":"b"}]}+ """), "us-east-1").path("ModelArn").asText();+ JsonNode tags = service.listTags(mapper.readTree("{\"ResourceArn\":\"" + arn + "\"}")).path("Tags");+ assertEquals(1, tags.size());+ assertEquals("a", tags.get(0).path("Key").asText());+ }++ @Test+ void trainingJobListingFiltersByNameAndStatus() throws Exception {+ SageMakerService service = service();+ for (String name : List.of("job-a", "job-b", "other")) {+ service.createTrainingJob(mapper.readTree("""+ {"TrainingJobName":"%s",+ "AlgorithmSpecification":{"TrainingImage":"busybox:stable"},+ "OutputDataConfig":{"S3OutputPath":"s3://bucket/out"}}+ """.formatted(name)), "us-east-1");+ }+ JsonNode filtered = service.listTrainingJobs(mapper.readTree("{\"NameContains\":\"job-\"}"), "us-east-1");+ assertEquals(2, filtered.path("TrainingJobSummaries").size());++ JsonNode byStatus = service.listTrainingJobs(mapper.readTree("{\"StatusEquals\":\"Completed\"}"), "us-east-1");+ assertEquals(0, byStatus.path("TrainingJobSummaries").size());+ }+++ @Test+ void s3UriParsingCoversPrefixesAndValidation() {+ S3Uri object = S3Uri.parse("s3://bucket/path/to/object");+ assertEquals("bucket", object.bucket());+ assertEquals("path/to/object", object.key());+ S3Uri bucketOnly = S3Uri.parse("s3://bucket");+ assertEquals("bucket", bucketOnly.bucket());+ assertEquals("", bucketOnly.key());+ assertThrows(IllegalArgumentException.class, () -> S3Uri.parse("http://bucket/key"));+ }++ private SageMakerService service() {+ return new SageMakerService(new InMemoryStorage<String, ModelResource>(),+ new InMemoryStorage<String, EndpointConfigResource>(),+ new InMemoryStorage<String, EndpointResource>(),+ new InMemoryStorage<String, TrainingJobResource>(),+ new RegionResolver("us-east-1", "000000000000"), mapper,+ mock(SageMakerEndpointManager.class), mock(SageMakerTrainingRunner.class));+ }+}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 010a662f05..41858aa365 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -44,6 +44,8 @@ floci:flush-interval-ms: 60000opensearch:flush-interval-ms: 60000+ sagemaker:+ flush-interval-ms: 60000batch:flush-interval-ms: 60000lightsail:@@ -315,6 +317,9 @@ floci:codebuild:enabled: true# docker-network: floci-network+ sagemaker:+ enabled: true+ # docker-network: floci-networkbatch:enabled: truerunner-mode: immediatediff --git a/tools/docs/services.yaml b/tools/docs/services.yamlindex ca8496125c..aaa5622b7d 100644--- a/tools/docs/services.yaml+++ b/tools/docs/services.yaml@@ -310,6 +310,10 @@ services:doc: docs/services/route53resolver.mdsources:- { path: src/main/java/io/github/hectorvent/floci/services/route53resolver/Route53ResolverJsonHandler.java, mode: switch }+ - service: sagemaker+ doc: docs/services/sagemaker.md+ sources:+ - { path: src/main/java/io/github/hectorvent/floci/services/sagemaker/SageMakerJsonHandler.java, mode: switch }- service: securityhubdoc: docs/services/securityhub.mdsources: