Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(ecr): PutImage and the layer API (#5784)

Davidc6551a47bcba
diff --git a/compatibility-tests/sdk-test-awscli/test/ecr.bats b/compatibility-tests/sdk-test-awscli/test/ecr.bats
index 96e33a7645..6422e98850 100644
--- a/compatibility-tests/sdk-test-awscli/test/ecr.bats
+++ b/compatibility-tests/sdk-test-awscli/test/ecr.bats
@@ -103,3 +103,40 @@ teardown() {
assert_failure
[[ "$output" == *"RepositoryNotFoundException"* ]]
}
+
+@test "ECR: an image pushed with the layer API retags with batch-get-image and put-image" {
+ aws_cmd ecr create-repository --repository-name "$REPO_NAME" >/dev/null
+ printf '{"architecture":"amd64","os":"linux"}' > "$BATS_TEST_TMPDIR/config"
+ printf 'floci layer bytes' > "$BATS_TEST_TMPDIR/layer"
+
+ for blob in config layer; do
+ file="$BATS_TEST_TMPDIR/$blob"
+ size=$(wc -c < "$file" | tr -d ' ')
+ digest="sha256:$(sha256sum "$file" | cut -d' ' -f1)"
+ upload=$(aws_cmd ecr initiate-layer-upload --repository-name "$REPO_NAME" | jq -r '.uploadId')
+ run aws_cmd ecr upload-layer-part --repository-name "$REPO_NAME" --upload-id "$upload" \
+ --part-first-byte 0 --part-last-byte $((size - 1)) --layer-part-blob "fileb://$file"
+ assert_success
+ run aws_cmd ecr complete-layer-upload --repository-name "$REPO_NAME" --upload-id "$upload" \
+ --layer-digests "$digest"
+ assert_success
+ [ "$(json_get "$output" '.layerDigest')" = "$digest" ]
+ eval "${blob}_digest=$digest ${blob}_size=$size"
+ done
+
+ manifest=$(printf '{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","size":%s,"digest":"%s"},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","size":%s,"digest":"%s"}]}' \
+ "$config_size" "$config_digest" "$layer_size" "$layer_digest")
+ run aws_cmd ecr put-image --repository-name "$REPO_NAME" --image-tag v1 --image-manifest "$manifest"
+ assert_success
+ digest=$(json_get "$output" '.image.imageId.imageDigest')
+
+ read_back=$(aws_cmd ecr batch-get-image --repository-name "$REPO_NAME" --image-ids imageTag=v1 \
+ --query 'images[0].imageManifest' --output text)
+ run aws_cmd ecr put-image --repository-name "$REPO_NAME" --image-tag release --image-manifest "$read_back"
+ assert_success
+ [ "$(json_get "$output" '.image.imageId.imageDigest')" = "$digest" ]
+
+ run aws_cmd ecr describe-images --repository-name "$REPO_NAME" --image-ids imageTag=release
+ assert_success
+ [ "$(json_get "$output" '.imageDetails[0].imageDigest')" = "$digest" ]
+}
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcrTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcrTest.java
index c16c37094b..2f77cc676b 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcrTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcrTest.java
@@ -7,11 +7,15 @@ import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
+import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.services.ecr.EcrClient;
import software.amazon.awssdk.services.ecr.model.*;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Base64;
+import java.util.HexFormat;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
@@ -238,4 +242,58 @@ class EcrTest {
}
}
}
+
+ @Test
+ @Order(13)
+ @DisplayName("An image pushed through the layer API and PutImage reads back and retags")
+ void pushThroughTheApiAndRetag() throws Exception {
+ String repo = "floci-it/api-push-" + System.nanoTime();
+ ecr.createRepository(b -> b.repositoryName(repo));
+ reposToCleanup.add(repo);
+
+ String config = "{\"architecture\":\"amd64\",\"os\":\"linux\"}";
+ String layer = "floci layer bytes";
+ String configDigest = uploadLayer(repo, config.getBytes(StandardCharsets.UTF_8));
+ String layerDigest = uploadLayer(repo, layer.getBytes(StandardCharsets.UTF_8));
+
+ BatchCheckLayerAvailabilityResponse check = ecr.batchCheckLayerAvailability(b -> b
+ .repositoryName(repo).layerDigests(configDigest, layerDigest));
+ assertThat(check.layers()).extracting(Layer::layerAvailability)
+ .containsOnly(LayerAvailability.AVAILABLE);
+ assertThat(check.failures()).isEmpty();
+
+ String manifest = "{\"schemaVersion\":2,\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\","
+ + "\"config\":{\"mediaType\":\"application/vnd.oci.image.config.v1+json\",\"size\":"
+ + config.length() + ",\"digest\":\"" + configDigest + "\"},"
+ + "\"layers\":[{\"mediaType\":\"application/vnd.oci.image.layer.v1.tar\",\"size\":"
+ + layer.length() + ",\"digest\":\"" + layerDigest + "\"}]}";
+ PutImageResponse put = ecr.putImage(b -> b.repositoryName(repo).imageTag("v1").imageManifest(manifest));
+ assertThat(put.image().imageId().imageDigest()).isEqualTo(sha256(manifest.getBytes(StandardCharsets.UTF_8)));
+
+ String read = ecr.batchGetImage(b -> b.repositoryName(repo)
+ .imageIds(ImageIdentifier.builder().imageTag("v1").build()))
+ .images().get(0).imageManifest();
+ ecr.putImage(b -> b.repositoryName(repo).imageTag("release").imageManifest(read));
+ assertThat(ecr.listImages(b -> b.repositoryName(repo)).imageIds())
+ .extracting(ImageIdentifier::imageTag).contains("v1", "release");
+ assertThatThrownBy(() -> ecr.putImage(b -> b.repositoryName(repo).imageTag("release").imageManifest(read)))
+ .isInstanceOf(ImageAlreadyExistsException.class);
+
+ String url = ecr.getDownloadUrlForLayer(b -> b.repositoryName(repo).layerDigest(layerDigest)).downloadUrl();
+ assertThat(url).endsWith("/v2/" + repo + "/blobs/" + layerDigest);
+ }
+
+ private static String uploadLayer(String repo, byte[] bytes) throws Exception {
+ String uploadId = ecr.initiateLayerUpload(b -> b.repositoryName(repo)).uploadId();
+ UploadLayerPartResponse part = ecr.uploadLayerPart(b -> b.repositoryName(repo).uploadId(uploadId)
+ .partFirstByte(0L).partLastByte((long) bytes.length - 1).layerPartBlob(SdkBytes.fromByteArray(bytes)));
+ assertThat(part.lastByteReceived()).isEqualTo(bytes.length - 1L);
+ String digest = sha256(bytes);
+ return ecr.completeLayerUpload(b -> b.repositoryName(repo).uploadId(uploadId).layerDigests(digest))
+ .layerDigest();
+ }
+
+ private static String sha256(byte[] bytes) throws Exception {
+ return "sha256:" + HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes));
+ }
}
diff --git a/docs/services/ecr.md b/docs/services/ecr.md
index 54df05647a..c8423059a9 100644
--- a/docs/services/ecr.md
+++ b/docs/services/ecr.md
@@ -21,6 +21,12 @@
| `DescribeImages` | Image metadata: digest, size, push timestamp, manifest media type |
| `BatchGetImage` | Fetch image manifests, honoring `acceptedMediaTypes` |
| `BatchDeleteImage` | Delete images by tag or digest |
+| `PutImage` | Write an image manifest under a tag, or under its digest, into the backing registry |
+| `BatchCheckLayerAvailability` | Report which layers a repository holds, with their sizes |
+| `GetDownloadUrlForLayer` | Return the layer's address on Floci's registry endpoint |
+| `InitiateLayerUpload` | Open a layer upload backed by a registry upload session |
+| `UploadLayerPart` | Send the next part of a layer upload |
+| `CompleteLayerUpload` | Store an uploaded layer under its digest |
| `PutImageTagMutability` | Set tag mutability and reject replacement pushes to immutable tags |
| `PutImageScanningConfiguration` | Set a repository's `scanOnPush` flag and return the updated configuration; a non-boolean value is rejected |
| `BatchGetRepositoryScanningConfiguration` | Report each repository's `scanOnPush` setting with a derived `scanFrequency` (`SCAN_ON_PUSH` or `MANUAL`); a missing repository fails the whole batch instead of appearing in `failures` |
@@ -41,7 +47,13 @@
- **Advertise TLS addresses.** Set `FLOCI_SERVICES_ECR_TLS_URI=true` together with `FLOCI_TLS_ENABLED=true` to return the TLS hostname in `repositoryUri` and an HTTPS `proxyEndpoint` from `GetAuthorizationToken`. This opt-in defaults to `false`. With `uri-style: path`, the URI becomes `localhost.floci.io:<port>/<account>/<region>/<repo>` and the login endpoint is `https://localhost.floci.io:<port>`. Existing repository records acquire the current URI when described or returned by deletion. If global TLS is disabled, the option has no effect. Clients must trust the Floci CA, or the configured certificate must cover the advertised names.
- **Storage cleanup.** `DeleteRepository --force` removes repository manifests, including untagged manifests. When it deletes the final repository, or when a non-retained runtime stops, Floci removes the named registry volume only in `memory` mode or when `prune-volumes-on-delete: true`.
- **Loopback URI scheme.** Repository URIs follow `<account>.dkr.ecr.<region>.localhost:<flociPort>/<repoName>`. RFC 6761 reserves `*.localhost` to resolve to the loopback address, and a Docker daemon with the classic image store auto-trusts loopback as an insecure registry, so `docker push` and `docker pull` work without daemon configuration. A daemon with the containerd image store, the default from Docker Engine 29, reaches only `localhost` and `127.0.0.1` over plain HTTP and takes `*.localhost` for an HTTPS registry: use the `path` URI style (`localhost:<flociPort>/<account>/<region>/<repo>`, `floci.services.ecr.uri-style: path`) there, as well as where `*.localhost` resolution misbehaves.
-- **Image tag mutability.** `IMMUTABLE` repositories allow the first manifest write for a tag and reject every replacement, including a replacement with the same manifest. The proxy forwards blob uploads and digest-addressed manifests unchanged.
+- **Image tag mutability.** `IMMUTABLE` repositories allow the first manifest write for a tag and reject every replacement, including a replacement with the same manifest. The proxy forwards blob uploads and digest-addressed manifests unchanged. `PutImage` follows the same rule and answers `ImageTagAlreadyExistsException`; a `docker push` and a `PutImage` of the same tag cannot both succeed.
+- **Pushing through the API.** `PutImage` and the layer API write to the same `registry:2` container as `docker push`, so an image written through the API is the one `docker pull` and `BatchGetImage` return. Retagging without Docker works as on AWS: read the manifest with `batch-get-image`, then write it with `put-image --image-tag <new>`.
+ - `PutImage` takes the media type from `imageManifestMediaType`, else from the manifest's own `mediaType`; a manifest with neither is `InvalidParameterException`, as on AWS. A tag outside ECR's rule (`[a-zA-Z0-9-_.]+`, at most 300 characters) is `InvalidParameterException` with ECR's message. The backing registry follows the OCI tag grammar, so Floci also refuses two kinds of tag ECR takes, with a message naming the limit: one starting with `.` or `-`, and one longer than 128 characters. A manifest whose layers the repository does not hold answers `LayersNotFoundException`; writing the manifest a tag already points at answers `ImageAlreadyExistsException`.
+ - Layer uploads map onto the registry's upload sessions. `InitiateLayerUpload` reports a `partSize` of 10 MiB. Each part must start at the byte after the last one received and can be at most 20 MiB, and every part but the last must be at least 5 MiB (`LayerPartTooSmallException`). `CompleteLayerUpload` checks the digest (`InvalidLayerException`) and refuses a layer the repository already holds (`LayerAlreadyExistsException`).
+ - Upload sessions live in memory: a restart ends them, and one left idle for a day is dropped.
+ - `GetDownloadUrlForLayer` returns `<scheme>://<repository URI host>/v2/<repo>/blobs/<digest>`, served by Floci's registry endpoint without credentials, in place of AWS's pre-signed S3 URL.
+ - `BatchCheckLayerAvailability` lists held layers as `AVAILABLE`, and a layer the repository does not hold under `failures` as `MissingLayerDigest`.
- **Authorization.** `GetAuthorizationToken` returns `Base64("AWS:floci")` plus a proxy endpoint. The backing `registry:2` runs without auth, so any `aws ecr get-login-password | docker login` succeeds.
- **Pull through cache rule configuration.** Pull-through cache rule actions persist AWS-compatible rule metadata with account and Region isolation. `UpdatePullThroughCacheRule` preserves fields omitted by the request and refreshes `updatedAt`. `ValidatePullThroughCacheRule` checks that the rule exists and reports it as valid; Floci does not contact the upstream registry or validate credentials. Rules do not proxy or cache images from the configured upstream registry.
- **Manifest format negotiation.** `BatchGetImage` forwards the caller's `acceptedMediaTypes` as the upstream `Accept` header. Modern OCI manifests (`application/vnd.oci.image.manifest.v1+json`) and Docker v2 schema 2 are both supported.
diff --git a/docs/services/index.md b/docs/services/index.md
index e341711361..173f0fe81c 100644
--- a/docs/services/index.md
+++ b/docs/services/index.md
@@ -86,7 +86,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac
| [EC2](ec2.md) | `POST /` with `Action=` param | EC2 Query | 198 |
| [Lightsail](lightsail.md) | `POST /` + `X-Amz-Target: Lightsail_20161128.*` | JSON 1.1 | 79 local responses; 161 recognized actions |
| [ACM](acm.md) | `POST /` + `X-Amz-Target: CertificateManager.*` | JSON 1.1 | 16 |
-| [ECR](ecr.md) | `POST /` + `X-Amz-Target: AmazonEC2ContainerRegistry_V20150921.*` (control plane) and `/v2/...` (data plane proxied to `registry:2`) | JSON 1.1 + OCI Distribution | 17 |
+| [ECR](ecr.md) | `POST /` + `X-Amz-Target: AmazonEC2ContainerRegistry_V20150921.*` (control plane) and `/v2/...` (data plane proxied to `registry:2`) | JSON 1.1 + OCI Distribution | 23 |
| [AWS Resource Groups](resource-groups.md) | `/groups`, `/get-group`, `/delete-group`, `/group-resources`, `/list-group-resources` | REST JSON | 5 |
| [Resource Groups Tagging API](resource-groups-tagging.md) | `POST /` + `X-Amz-Target: ResourceGroupsTaggingAPI_20170126.*` | JSON 1.1 | 5 |
| [Resource Explorer](resource-explorer.md) | `POST /{OperationName}`, rewritten to `/_re2/*` for the four paths S3 Vectors also claims | REST JSON | 32 |
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecr/EcrJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/ecr/EcrJsonHandler.java
index 47b944208c..585d668f08 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecr/EcrJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecr/EcrJsonHandler.java
@@ -20,6 +20,7 @@ import jakarta.inject.Inject;
import jakarta.ws.rs.core.Response;
import java.util.ArrayList;
+import java.util.Base64;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -59,6 +60,12 @@ public class EcrJsonHandler {
case "DescribeImages" -> handleDescribeImages(request, region);
case "BatchGetImage" -> handleBatchGetImage(request, region);
case "BatchDeleteImage" -> handleBatchDeleteImage(request, region);
+ case "PutImage" -> handlePutImage(request, region);
+ case "BatchCheckLayerAvailability" -> handleBatchCheckLayerAvailability(request, region);
+ case "GetDownloadUrlForLayer" -> handleGetDownloadUrlForLayer(request, region);
+ case "InitiateLayerUpload" -> handleInitiateLayerUpload(request, region);
+ case "UploadLayerPart" -> handleUploadLayerPart(request, region);
+ case "CompleteLayerUpload" -> handleCompleteLayerUpload(request, region);
case "PutImageTagMutability" -> handlePutImageTagMutability(request, region);
case "TagResource" -> handleTagResource(request, region);
case "UntagResource" -> handleUntagResource(request, region);
@@ -377,6 +384,107 @@ public class EcrJsonHandler {
// Tag mutability + resource tags
// ============================================================
+ private Response handlePutImage(JsonNode request, String region) {
+ Image img = service.putImage(
+ request.path("repositoryName").asText(null),
+ request.path("imageManifest").asText(null),
+ request.path("imageManifestMediaType").asText(null),
+ request.path("imageTag").asText(null),
+ request.path("imageDigest").asText(null),
+ request.path("registryId").asText(null),
+ region);
+ ObjectNode n = objectMapper.createObjectNode();
+ n.put("registryId", img.getRegistryId());
+ n.put("repositoryName", img.getRepositoryName());
+ n.set("imageId", buildImageIdentifier(img.getImageId()));
+ n.put("imageManifest", img.getImageManifest());
+ n.put("imageManifestMediaType", img.getImageManifestMediaType());
+ ObjectNode response = objectMapper.createObjectNode();
+ response.set("image", n);
+ return Response.ok(response).build();
+ }
+
+ private Response handleBatchCheckLayerAvailability(JsonNode request, String region) {
+ EcrService.BatchCheckLayerAvailabilityResult result = service.batchCheckLayerAvailability(
+ request.path("repositoryName").asText(null),
+ parseStringList(request.path("layerDigests")),
+ request.path("registryId").asText(null),
+ region);
+ ObjectNode response = objectMapper.createObjectNode();
+ ArrayNode layers = response.putArray("layers");
+ for (EcrService.LayerAvailability layer : result.layers()) {
+ ObjectNode n = layers.addObject();
+ n.put("layerDigest", layer.layerDigest());
+ n.put("layerAvailability", layer.availability());
+ n.put("layerSize", layer.layerSize());
+ }
+ ArrayNode failures = response.putArray("failures");
+ for (EcrService.LayerFailure failure : result.failures()) {
+ ObjectNode n = failures.addObject();
+ if (failure.layerDigest() != null) {
+ n.put("layerDigest", failure.layerDigest());
+ }
+ n.put("failureCode", failure.failureCode());
+ n.put("failureReason", failure.failureReason());
+ }
+ return Response.ok(response).build();
+ }
+
+ private Response handleGetDownloadUrlForLayer(JsonNode request, String region) {
+ String digest = request.path("layerDigest").asText(null);
+ String url = service.getDownloadUrlForLayer(
+ request.path("repositoryName").asText(null),
+ digest,
+ request.path("registryId").asText(null),
+ region);
+ ObjectNode response = objectMapper.createObjectNode();
+ response.put("downloadUrl", url);
+ response.put("layerDigest", digest);
+ return Response.ok(response).build();
+ }
+
+ private Response handleInitiateLayerUpload(JsonNode request, String region) {
+ EcrService.LayerUpload upload = service.initiateLayerUpload(
+ request.path("repositoryName").asText(null),
+ request.path("registryId").asText(null),
+ region);
+ ObjectNode response = objectMapper.createObjectNode();
+ response.put("uploadId", upload.uploadId());
+ response.put("partSize", EcrLayerUploads.PART_SIZE);
+ return Response.ok(response).build();
+ }
+
+ private Response handleUploadLayerPart(JsonNode request, String region) {
+ String repositoryName = request.path("repositoryName").asText(null);
+ String uploadId = request.path("uploadId").asText(null);
+ String registryId = request.path("registryId").asText(null);
+ long lastByte = service.uploadLayerPart(repositoryName, uploadId,
+ parseLong(request.path("partFirstByte"), "partFirstByte"),
+ parseLong(request.path("partLastByte"), "partLastByte"),
+ parseBlob(request.path("layerPartBlob")),
+ registryId, region);
+ ObjectNode response = objectMapper.createObjectNode();
+ response.put("registryId", service.registryIdFor(registryId));
+ response.put("repositoryName", repositoryName);
+ response.put("uploadId", uploadId);
+ response.put("lastByteReceived", lastByte);
+ return Response.ok(response).build();
+ }
+
+ private Response handleCompleteLayerUpload(JsonNode request, String region) {
+ String repositoryName = request.path("repositoryName").asText(null);
+ String uploadId = request.path("uploadId").asText(null);
+ String registryId = request.path("registryId").asText(null);
+ String digest = service.completeLayerUpload(repositoryName, uploadId,
+ parseStringList(request.path("layerDigests")), registryId, region);
+ ObjectNode response = objectMapper.createObjectNode();
+ response.put("registryId", service.registryIdFor(registryId));
+ response.put("repositoryName", repositoryName);
+ response.put("uploadId", uploadId);
+ response.put("layerDigest", digest);
+ return Response.ok(response).build();
+ }
+
private Response handlePutImageTagMutability(JsonNode request, String region) {
String repo = request.path("repositoryName").asText(null);
String registryId = request.path("registryId").asText(null);
@@ -617,6 +725,27 @@ public class EcrJsonHandler {
return out;
}
+ private static Long parseLong(JsonNode node, String name) {
+ if (node == null || node.isMissingNode() || node.isNull()) {
+ return null;
+ }
+ if (!node.isIntegralNumber() || !node.canConvertToLong()) {
+ throw new AwsException("InvalidParameterException", name + " must be an integer.", 400);
+ }
+ return node.longValue();
+ }
+
+ private static byte[] parseBlob(JsonNode node) {
+ if (node == null || node.isMissingNode() || node.isNull()) {
+ return null;
+ }
+ try {
+ return Base64.getDecoder().decode(node.asText());
+ } catch (IllegalArgumentException e) {
+ throw new AwsException("InvalidParameterException", "layerPartBlob must be base64-encoded.", 400);
+ }
+ }
+
private static Integer parseMaxResults(JsonNode node) {
if (node == null || node.isMissingNode() || node.isNull()) {
return null;
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecr/EcrLayerUploads.java b/src/main/java/io/github/hectorvent/floci/services/ecr/EcrLayerUploads.java
new file mode 100644
index 0000000000..5fd90d7e01
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ecr/EcrLayerUploads.java
@@ -0,0 +1,165 @@
+package io.github.hectorvent.floci.services.ecr;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Consumer;
+
+/**
+ * The layer uploads opened by {@code InitiateLayerUpload}, each tied to the {@code registry:2}
+ * upload session that holds its bytes. Sessions live in memory, as the registry's own do, and one
+ * left idle for a day is dropped, along with its registry upload session.
+ */
+class EcrLayerUploads {
+
+ /** The part size {@code InitiateLayerUpload} reports. */
+ static final long PART_SIZE = 10L * 1024 * 1024;
+ /** The largest part {@code UploadLayerPart} takes, as the API reference gives it. */
+ static final long MAX_PART_SIZE = 20L * 1024 * 1024;
+ /** The smallest part other than the last, as {@code LayerPartTooSmallException} gives it. */
+ static final long MIN_PART_SIZE = 5L * 1024 * 1024;
+ static final Duration IDLE_LIMIT = Duration.ofDays(1);
+
+ /** One open upload: the repository it belongs to, the registry URL for its next chunk and its parts so far. */
+ static final class Session {
+ private final String scope;
+ private String location;
+ private long received;
+ private final List<Long> partSizes = new ArrayList<>();
+ private Instant lastTouched;
+
+ Session(String scope, String location, Instant now) {
+ this.scope = scope;
+ this.location = location;
+ this.lastTouched = now;
+ }
+
+ String location() {
+ return location;
+ }
+
+ long received() {
+ return received;
+ }
+
+ List<Long> partSizes() {
+ return partSizes;
+ }
+
+ /** Records a part the registry accepted, and the URL it gave for the next one. */
+ void accept(long size, String nextLocation, Instant now) {
+ partSizes.add(size);
+ received += size;
+ location = nextLocation;
+ lastTouched = now;
+ }
+ }
+
+ private final Map<String, Session> sessions = new ConcurrentHashMap<>();
+ private final Clock clock;
+ /** Cancels a dropped session's registry upload, given its location. */
+ private final Consumer<String> discard;
+
+ EcrLayerUploads(Consumer<String> discard) {
+ this(Clock.systemUTC(), discard);
+ }
+
+ EcrLayerUploads(Clock clock, Consumer<String> discard) {
+ this.clock = clock;
+ this.discard = discard;
+ }
+
+ Instant now() {
+ return clock.instant();
+ }
+
+ /** Opens an upload for the repository {@code scope} names, returning its upload ID. */
+ String open(String scope, String location) {
+ Instant now = clock.instant();
+ sessions.forEach((id, session) -> {
+ if (idle(session, now)) {
+ drop(id, session);
+ }
+ });
+ String uploadId = UUID.randomUUID().toString();
+ sessions.put(uploadId, new Session(scope, location, now));
+ return uploadId;
+ }
+
+ /**
+ * The open upload {@code uploadId} names in this repository, or {@code UploadNotFoundException},
+ * which is also the answer for one idle for a day even if no other upload has opened since.
+ */
+ Session require(String scope, String uploadId, String repositoryName) {
+ Session session = uploadId == null ? null : sessions.get(uploadId);
+ if (session != null && idle(session, clock.instant())) {
+ drop(uploadId, session);
+ session = null;
+ }
+ if (session == null || !session.scope.equals(scope)) {
+ throw new AwsException("UploadNotFoundException",
+ "Upload with id '" + uploadId + "' is not found for repository '" + repositoryName + "'", 400);
+ }
+ return session;
+ }
+
+ void close(String uploadId) {
+ sessions.remove(uploadId);
+ }
+
+ private static boolean idle(Session session, Instant now) {
+ return session.lastTouched.plus(IDLE_LIMIT).isBefore(now);
+ }
+
+ private void drop(String uploadId, Session session) {
+ if (sessions.remove(uploadId, session)) {
+ discard.accept(session.location);
+ }
+ }
+
+ /**
+ * A part must follow the last byte received, match its stated range and fit the size limit.
+ * The error carries the upload and its last valid byte, as {@code InvalidLayerPartException} does.
+ */
+ static void checkPart(Session session, long firstByte, long lastByte, int length,
+ String registryId, String repositoryName, String uploadId) {
+ long expected = session.received;
+ String message = null;
+ if (firstByte != expected) {
+ message = "The part's first byte " + firstByte + " is not consecutive to the last byte "
+ + "received; the next part must start at byte " + expected + ".";
+ } else if (length == 0 || lastByte - firstByte + 1 != length) {
+ message = "The part holds " + length + " bytes, but the range " + firstByte + "-" + lastByte
+ + " names " + Math.max(0, lastByte - firstByte + 1) + ".";
+ } else if (length > MAX_PART_SIZE) {
+ message = "A layer part can be at most " + MAX_PART_SIZE + " bytes.";
+ }
+ if (message != null) {
+ throw new AwsException("InvalidLayerPartException", message, 400, Map.of(
+ "registryId", registryId,
+ "repositoryName", repositoryName,
+ "uploadId", uploadId,
+ "lastValidByteReceived", Math.max(0, expected - 1)));
+ }
+ }
+
+ /** An upload completes only with at least one part, and every part but the last at least 5 MiB. */
+ static void checkComplete(Session session) {
+ List<Long> parts = session.partSizes;
+ if (parts.isEmpty()) {
+ throw new AwsException("EmptyUploadException", "The layer upload does not contain any layer parts.", 400);
+ }
+ for (int i = 0; i < parts.size() - 1; i++) {
+ if (parts.get(i) < MIN_PART_SIZE) {
+ throw new AwsException("LayerPartTooSmallException", "Layer parts must be at least 5 MiB in size.", 400);
+ }
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecr/EcrService.java b/src/main/java/io/github/hectorvent/floci/services/ecr/EcrService.java
index b9166d2da1..0e32bed461 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecr/EcrService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecr/EcrService.java
@@ -1,10 +1,13 @@
package io.github.hectorvent.floci.services.ecr;
import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.AwsPartitions;
+import io.github.hectorvent.floci.core.common.Digests;
import io.github.hectorvent.floci.core.common.PaginatedResult;
import io.github.hectorvent.floci.core.common.Pagination;
import io.github.hectorvent.floci.core.common.RegionResolver;
@@ -36,6 +39,8 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
import java.util.regex.Pattern;
@ApplicationScoped
@@ -55,6 +60,13 @@ public class EcrService implements ResourceProvider {
"(?:[a-z0-9]+(?:(?:\\.|_|__|-+)[a-z0-9]+)*(?:/[a-z0-9]+"
+ "(?:(?:\\.|_|__|-+)[a-z0-9]+)*)*/?|ROOT)");
private static final Pattern REGISTRY_ID = Pattern.compile("[0-9]{12}");
+ private static final Pattern LAYER_DIGEST = Pattern.compile("[a-zA-Z0-9-_+.]+:[a-fA-F0-9]+");
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ /** The OCI distribution spec's tag grammar, which the registry's manifest URL takes. */
+ /** ECR's imageTag rule, which the service model bounds at 300 characters. */
+ private static final Pattern ECR_IMAGE_TAG = Pattern.compile("[a-zA-Z0-9_.-]+");
+ /** The OCI distribution spec's tag grammar, which the backing registry's manifest URL takes. */
+ private static final Pattern REGISTRY_IMAGE_TAG = Pattern.compile("[A-Za-z0-9_][A-Za-z0-9._-]{0,127}");
private static final Pattern CREDENTIAL_ARN = Pattern.compile(
"arn:" + AwsArnUtils.PARTITION_REGEX + ":secretsmanager:[a-zA-Z0-9-:]+:secret:ecr-pullthroughcache/"
+ "[a-zA-Z0-9/_+=.@-]+");
@@ -71,6 +83,8 @@ public class EcrService implements ResourceProvider {
private final EcrRegistryManager registryManager;
private final EmulatorConfig config;
private final RegionResolver regionResolver;
+ private final EcrLayerUploads layerUploads = new EcrLayerUploads(this::discardUpload);
+ private final Set<String> inFlightImmutableTags = ConcurrentHashMap.newKeySet();
@Inject
public EcrService(StorageFactory factory,
@@ -621,6 +635,346 @@ public class EcrService implements ResourceProvider {
return new BatchDeleteImageResult(deleted, failures);
}
+ // ============================================================
+ // Image push through the API (PutImage and the layer API)
+ // ============================================================
+
+ public record LayerAvailability(String layerDigest, String availability, Long layerSize) {}
+ public record LayerFailure(String layerDigest, String failureCode, String failureReason) {}
+ public record BatchCheckLayerAvailabilityResult(List<LayerAvailability> layers, List<LayerFailure> failures) {}
+ public record LayerUpload(String registryId, String repositoryName, String uploadId) {}
+
+ /**
+ * Writes a manifest to the registry under its tag, or under its digest when no tag is given, so
+ * {@code docker pull} and {@code BatchGetImage} see it as they see a pushed image.
+ */
+ public Image putImage(String repositoryName, String imageManifest, String imageManifestMediaType,
+ String imageTag, String imageDigest, String registryId, String region) {
+ Repository repo = requireRepo(repositoryName, registryId, region);
+ if (imageManifest == null || imageManifest.isEmpty()) {
+ throw new AwsException("InvalidParameterException", "imageManifest must not be empty", 400);
+ }
+ if (imageTag != null) {
+ requireImageTag(imageTag);
+ }
+ String mediaType = manifestMediaType(imageManifest, imageManifestMediaType);
+ String digest = "sha256:" + Digests.sha256Hex(imageManifest.getBytes(StandardCharsets.UTF_8));
+ if (imageDigest != null && !imageDigest.isEmpty() && !imageDigest.equals(digest)) {
+ throw new AwsException("ImageDigestDoesNotMatchException",
+ "Invalid parameter at 'imageDigest' failed to satisfy constraint: 'specified digest "
+ + imageDigest + " does not match the calculated digest " + digest + "'", 400);
+ }
+ requireRegistry();
+ String storage = registryRepositoryName(repo, region);
+ RegistryHttpClient http = registryManager.httpClient();
+
+ String guarded = imageTag != null && "IMMUTABLE".equals(repo.getImageTagMutability())
+ ? immutableTagKey(repo.getRegistryId(), region, repositoryName, imageTag) : null;
+ if (guarded != null && !claimImmutableTag(guarded)) {
+ throw imageTagAlreadyExists(imageTag, repositoryName);
+ }
+ try {
+ return callRegistry(() -> writeManifest(repo, repositoryName, storage, http, imageManifest, mediaType,
+ imageTag, digest, guarded, region), EcrService::putImageError);
+ } finally {
+ if (guarded != null) {
+ releaseImmutableTag(guarded);
+ }
+ }
+ }
+
+ private Image writeManifest(Repository repo, String repositoryName, String storage, RegistryHttpClient http,
+ String imageManifest, String mediaType, String imageTag, String digest,
+ String guarded, String region) throws Exception {
+ String existing = http.headManifestDigestStrict(storage, imageTag != null ? imageTag : digest, null);
+ if (digest.equals(existing)) {
+ throw new AwsException("ImageAlreadyExistsException",
+ "Image with digest '" + digest + "'" + (imageTag != null ? " and tag '" + imageTag + "'" : "")
+ + " already exists in the repository with name '" + repositoryName
+ + "' in registry with id '" + repo.getRegistryId() + "'", 400);
+ }
+ if (existing != null && guarded != null) {
+ throw imageTagAlreadyExists(imageTag, repositoryName);
+ }
+ String stored = http.putManifest(storage, imageTag != null ? imageTag : digest, mediaType, imageManifest);
+ String finalDigest = stored != null ? stored : digest;
+ // A retag keeps the image's first push time, as DescribeImages records it.
+ String metaKey = imageMetaKey(region, repo.getRegistryId(), repositoryName, finalDigest);
+ if (imageMetaStore.get(metaKey).isEmpty()) {
+ imageMetaStore.put(metaKey, new ImageMetadata(finalDigest, Instant.now()));
+ }
+
+ Image image = new Image();
+ image.setRegistryId(repo.getRegistryId());
+ image.setRepositoryName(repositoryName);
+ image.setImageId(new ImageIdentifier(imageTag, finalDigest));
+ image.setImageManifest(imageManifest);
+ image.setImageManifestMediaType(mediaType);
+ return image;
+ }
+
+ /**
+ * Refuses a tag ECR refuses, with ECR's message, then a tag ECR takes but the backing registry
+ * cannot hold: one starting with '.' or '-', or longer than 128 characters.
+ */
+ private static void requireImageTag(String imageTag) {
+ if (imageTag.length() > 300) {
+ throw new AwsException("InvalidParameterException", "Invalid parameter at 'imageTag' failed to satisfy"
+ + " constraint: 'Member must have length less than or equal to 300'", 400);
+ }
+ if (!ECR_IMAGE_TAG.matcher(imageTag).matches()) {
+ throw new AwsException("InvalidParameterException", "Invalid parameter at 'imageTag' failed to satisfy"
+ + " constraint: 'must satisfy regular expression '[a-zA-Z0-9-_.]+''", 400);
+ }
+ if (!REGISTRY_IMAGE_TAG.matcher(imageTag).matches()) {
+ throw new AwsException("InvalidParameterException", "Floci's backing registry cannot hold the tag '"
+ + imageTag + "': it must start with a letter, digit or underscore and be at most 128"
+ + " characters", 400);
+ }
+ }
+
+ /** Reports which of the given layers the repository holds, with their sizes. */
+ public BatchCheckLayerAvailabilityResult batchCheckLayerAvailability(String repositoryName,
+ List<String> layerDigests,
+ String registryId,
+ String region) {
+ Repository repo = requireRepo(repositoryName, registryId, region);
+ if (layerDigests == null || layerDigests.isEmpty() || layerDigests.size() > 100) {
+ throw new AwsException("InvalidParameterException", "layerDigests must hold 1 to 100 digests", 400);
+ }
+ requireRegistry();
+ String storage = registryRepositoryName(repo, region);
+ RegistryHttpClient http = registryManager.httpClient();
+ List<LayerAvailability> layers = new ArrayList<>();
+ List<LayerFailure> failures = new ArrayList<>();
+ for (String digest : layerDigests) {
+ if (digest == null || !LAYER_DIGEST.matcher(digest).matches()) {
+ failures.add(new LayerFailure(digest, "InvalidLayerDigest", "Invalid layer digest"));
+ continue;
+ }
+ Long size = headBlob(http, storage, digest);
+ if (size == null) {
+ failures.add(new LayerFailure(digest, "MissingLayerDigest", "Could not find layer"));
+ } else {
+ layers.add(new LayerAvailability(digest, "AVAILABLE", size));
+ }
+ }
+ return new BatchCheckLayerAvailabilityResult(layers, failures);
+ }
+
+ /** The address a layer can be fetched from: Floci's registry endpoint, which needs no credentials. */
+ public String getDownloadUrlForLayer(String repositoryName, String layerDigest, String registryId, String region) {
+ Repository repo = requireRepo(repositoryName, registryId, region);
+ requireLayerDigest(layerDigest);
+ requireRegistry();
+ Long size = headBlob(registryManager.httpClient(), registryRepositoryName(repo, region), layerDigest);
+ if (size == null) {
+ throw layersNotFound(layerDigest, repositoryName, repo.getRegistryId());
+ }
+ return registryManager.blobUrl(repo.getRepositoryUri(), layerDigest);
+ }
+
+ /** Cancels the registry upload session of a layer upload left idle for a day. */
+ private void discardUpload(String location) {
+ registryManager.httpClient().cancelUpload(location);
+ }
+
+ /** Opens a registry upload session for one layer. */
+ public LayerUpload initiateLayerUpload(String repositoryName, String registryId, String region) {
+ Repository repo = requireRepo(repositoryName, registryId, region);
+ requireRegistry();
+ String location = callRegistry(
+ () -> registryManager.httpClient().startUpload(registryRepositoryName(repo, region)),
+ EcrService::registryUnavailable);
+ String uploadId = layerUploads.open(key(region, repo.getRegistryId(), repositoryName), location);
+ return new LayerUpload(repo.getRegistryId(), repositoryName, uploadId);
+ }
+
+ /** Sends one part of a layer, returning the last byte received. */
+ public long uploadLayerPart(String repositoryName, String uploadId, Long partFirstByte, Long partLastByte,
+ byte[] layerPartBlob, String registryId, String region) {
+ Repository repo = requireRepo(repositoryName, registryId, region);
+ if (partFirstByte == null || partLastByte == null || layerPartBlob == null
+ || partFirstByte < 0 || partLastByte < 0) {
+ throw new AwsException("InvalidParameterException",
+ "partFirstByte, partLastByte and layerPartBlob are required and must not be negative", 400);
+ }
+ EcrLayerUploads.Session session = layerUploads.require(
+ key(region, repo.getRegistryId(), repositoryName), uploadId, repositoryName);
+ synchronized (session) {
+ EcrLayerUploads.checkPart(session, partFirstByte, partLastByte, layerPartBlob.length,
+ repo.getRegistryId(), repositoryName, uploadId);
+ requireRegistry();
+ String next = callRegistry(
+ () -> registryManager.httpClient().patchUpload(session.location(), layerPartBlob, partFirstByte),
+ e -> uploadError(e, uploadId, repositoryName));
+ session.accept(layerPartBlob.length, next, layerUploads.now());
+ return partLastByte;
+ }
+ }
+
+ /** Closes an upload, storing its bytes under the digest given, and returns that digest. */
+ public String completeLayerUpload(String repositoryName, String uploadId, List<String> layerDigests,
+ String registryId, String region) {
+ Repository repo = requireRepo(repositoryName, registryId, region);
+ if (layerDigests == null || layerDigests.isEmpty() || layerDigests.size() > 100) {
+ throw new AwsException("InvalidParameterException", "layerDigests must hold 1 to 100 digests", 400);
+ }
+ // An upload holds one layer, so every digest given must name it.
+ String digest = layerDigests.get(0);
+ for (String d : layerDigests) {
+ requireLayerDigest(d);
+ if (!digest.equals(d)) {
+ throw new AwsException("InvalidParameterException",
+ "layerDigests must all name the one layer the upload holds", 400);
+ }
+ }
+ EcrLayerUploads.Session session = layerUploads.require(
+ key(region, repo.getRegistryId(), repositoryName), uploadId, repositoryName);
+ synchronized (session) {
+ EcrLayerUploads.checkComplete(session);
+ requireRegistry();
+ RegistryHttpClient http = registryManager.httpClient();
+ String storage = registryRepositoryName(repo, region);
+ if (headBlob(http, storage, digest) != null) {
+ http.cancelUpload(session.location());
+ layerUploads.close(uploadId);
+ throw new AwsException("LayerAlreadyExistsException",
+ "Layer " + digest + " already exists in the repository with name '" + repositoryName
+ + "' in registry with id '" + repo.getRegistryId() + "'", 400);
+ }
+ callRegistry(() -> {
+ http.finishUpload(session.location(), digest);
+ return null;
+ }, e -> {
+ layerUploads.close(uploadId);
+ if ("DIGEST_INVALID".equals(e.code())) {
+ return new AwsException("InvalidLayerException",
+ "The layer digest calculated for the upload does not match " + digest, 400);
+ }
+ return uploadError(e, uploadId, repositoryName);
+ });
+ layerUploads.close(uploadId);
+ return digest;
+ }
+ }
+
+ /** The registry ID a request names, or the caller's account when it names none. */
+ public String registryIdFor(String registryId) {
+ return effectiveAccount(registryId);
+ }
+
+ /** The key both the registry data plane and {@code PutImage} hold while writing an immutable tag. */
+ public static String immutableTagKey(String account, String region, String repositoryName, String tag) {
+ return account + "/" + region + "/" + repositoryName + ":" + tag;
+ }
+
+ /** Claims an immutable tag for one write; false if another write holds it. */
+ public boolean claimImmutableTag(String key) {
+ return inFlightImmutableTags.add(key);
+ }
+
+ public void releaseImmutableTag(String key) {
+ inFlightImmutableTags.remove(key);
+ }
+
+ /** The media type sent, else the manifest's own {@code mediaType}; with neither, the request is refused. */
+ private static String manifestMediaType(String manifest, String given) {
+ JsonNode root;
+ try {
+ root = MAPPER.readTree(manifest);
+ } catch (Exception e) {
+ root = null;
+ }
+ if (root == null || !root.isObject()) {
+ throw new AwsException("InvalidParameterException", "imageManifest is not a valid JSON manifest", 400);
+ }
+ if (given != null && !given.isEmpty()) {
+ return given;
+ }
+ String own = root.path("mediaType").asText("");
+ if (own.isEmpty()) {
+ throw new AwsException("InvalidParameterException",
+ "imageManifestMediaType is required when the manifest has no mediaType", 400);
+ }
+ return own;
+ }
+
+ private Long headBlob(RegistryHttpClient http, String storage, String digest) {
+ return callRegistry(() -> http.headBlob(storage, digest), EcrService::registryUnavailable);
+ }
+
+ /** One call to the backing registry. */
+ @FunctionalInterface
+ private interface RegistryCall<T> {
+ T call() throws Exception;
+ }
+
+ /**
+ * Runs a call to the backing registry, answering a refusal the registry gives with what
+ * {@code refused} maps it to and any other failure with {@code ServerException}.
+ */
+ private static <T> T callRegistry(RegistryCall<T> call,
+ Function<RegistryHttpClient.RegistryErrorException, AwsException> refused) {
+ try {
+ return call.call();
+ } catch (AwsException e) {
+ throw e;
+ } catch (RegistryHttpClient.RegistryErrorException e) {
+ throw refused.apply(e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw registryUnavailable(e);
+ } catch (Exception e) {
+ throw registryUnavailable(e);
+ }
+ }
+
+ private static void requireLayerDigest(String digest) {
+ if (digest == null || !LAYER_DIGEST.matcher(digest).matches()) {
+ throw new AwsException("InvalidParameterException", "Invalid layer digest: " + digest, 400);
+ }
+ }
+
+ private static AwsException putImageError(RegistryHttpClient.RegistryErrorException e) {
+ String code = e.code() == null ? "" : e.code();
+ return switch (code) {
+ case "MANIFEST_BLOB_UNKNOWN", "BLOB_UNKNOWN" -> new AwsException("LayersNotFoundException",
+ "The image references layers that are not in the repository: " + e.getMessage(), 400);
+ case "MANIFEST_INVALID", "MANIFEST_UNVERIFIED", "DIGEST_INVALID", "TAG_INVALID", "NAME_INVALID" ->
+ new AwsException("InvalidParameterException", "The registry refused the manifest: " + e.getMessage(), 400);
+ default -> registryUnavailable(e);
+ };
+ }
+
+ private static AwsException uploadError(RegistryHttpClient.RegistryErrorException e, String uploadId,
+ String repositoryName) {
+ if ("BLOB_UPLOAD_UNKNOWN".equals(e.code()) || e.status() == 404) {
+ return new AwsException("UploadNotFoundException",
+ "Upload with id '" + uploadId + "' is not found for repository '" + repositoryName + "'", 400);
+ }
+ if ("BLOB_UPLOAD_INVALID".equals(e.code()) || e.status() == 416) {
+ return new AwsException("InvalidLayerPartException", "The registry refused the part: " + e.getMessage(), 400);
+ }
+ return registryUnavailable(e);
+ }
+
+ private static AwsException imageTagAlreadyExists(String tag, String repositoryName) {
+ return new AwsException("ImageTagAlreadyExistsException",
+ "The image tag '" + tag + "' already exists in the '" + repositoryName
+ + "' repository and cannot be overwritten because the repository is immutable.", 400);
+ }
+
+ private static AwsException layersNotFound(String digest, String repositoryName, String registryId) {
+ return new AwsException("LayersNotFoundException",
+ "The specified layer " + digest + " could not be found in the repository with name '"
+ + repositoryName + "' in the registry with id '" + registryId + "'", 400);
+ }
+
+ private static AwsException registryUnavailable(Exception e) {
+ return new AwsException("ServerException", "The ECR backing registry failed: " + e.getMessage(), 500);
+ }
+
// ============================================================
// Tag mutability + resource tags + policies (metadata round-trip)
// ============================================================
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlane.java b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlane.java
index 2b2b7fdc6a..aaf9cf0c35 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlane.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlane.java
@@ -23,7 +23,6 @@ import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -46,7 +45,6 @@ public class EcrRegistryDataPlane {
private final EcrService ecrService;
private final EmulatorConfig config;
private final Vertx vertx;
- private final Set<String> inFlightImmutableTags = ConcurrentHashMap.newKeySet();
@Inject
public EcrRegistryDataPlane(EcrRegistryManager registryManager,
@@ -82,7 +80,7 @@ public class EcrRegistryDataPlane {
registryRequest.repositoryName(), registryRequest.accountId(), registryRequest.region()));
}
String immutableTag = immutableTag(context, registryRequest);
- if (immutableTag != null && !inFlightImmutableTags.add(immutableTag)) {
+ if (immutableTag != null && !ecrService.claimImmutableTag(immutableTag)) {
context.request().resume();
imageTagAlreadyExists(context, registryRequest);
return;
@@ -103,7 +101,7 @@ public class EcrRegistryDataPlane {
|| !ecrService.isImageTagImmutable(request.repositoryName(), request.accountId(), request.region())) {
return null;
}
- return request.accountId() + "/" + request.region() + "/" + request.repositoryName() + ":" + request.tag();
+ return EcrService.immutableTagKey(request.accountId(), request.region(), request.repositoryName(), request.tag());
}
private void checkTagAndProxy(RoutingContext context, RegistryRequest request, Exchange exchange) {
@@ -242,7 +240,7 @@ public class EcrRegistryDataPlane {
private void release(String immutableTag) {
if (immutableTag != null) {
- inFlightImmutableTags.remove(immutableTag);
+ ecrService.releaseImmutableTag(immutableTag);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java
index 066aeb69ef..bb90d819be 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java
@@ -180,6 +180,18 @@ public class EcrRegistryManager {
return config.services().ecr().tlsUri() && config.tls().enabled();
}
+ /**
+ * The address a blob can be fetched from through Floci's registry endpoint: the repository URI's
+ * host, with {@code /v2/<repository>/blobs/<digest>}, over the scheme the proxy endpoint uses.
+ */
+ public String blobUrl(String repositoryUri, String digest) {
+ int slash = repositoryUri.indexOf('/');
+ String endpoint = getProxyEndpoint();
+ String scheme = endpoint.substring(0, endpoint.indexOf("://"));
+ return scheme + "://" + repositoryUri.substring(0, slash) + "/v2/" + repositoryUri.substring(slash + 1)
+ + "/blobs/" + digest;
+ }
+
/**
* Returns the proxy endpoint a docker daemon should log into for any ECR repo.
*
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/RegistryHttpClient.java b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/RegistryHttpClient.java
index 1a65c02a10..007390e07f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/RegistryHttpClient.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/RegistryHttpClient.java
@@ -6,9 +6,11 @@ import org.jboss.logging.Logger;
import java.io.IOException;
import java.net.URI;
+import java.net.URLEncoder;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
+import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
@@ -224,6 +226,146 @@ public class RegistryHttpClient {
return true;
}
+ /**
+ * A registry refusal, carrying the first {@code errors[].code} of the registry's error body
+ * (for example {@code MANIFEST_BLOB_UNKNOWN}) so the ECR API can answer with its own error.
+ */
+ public static class RegistryErrorException extends IOException {
+ private final int status;
+ private final String code;
+
+ public RegistryErrorException(int status, String code, String message) {
+ super(message);
+ this.status = status;
+ this.code = code;
+ }
+
+ public int status() {
+ return status;
+ }
+
+ public String code() {
+ return code;
+ }
+ }
+
+ /** PUT a manifest under a tag or digest, returning the digest the registry stored it under. */
+ public String putManifest(String name, String reference, String mediaType, String body)
+ throws IOException, InterruptedException {
+ HttpResponse<String> resp = http.send(
+ HttpRequest.newBuilder(URI.create(baseUrl + "/v2/" + name + "/manifests/" + reference))
+ .timeout(Duration.ofSeconds(30))
+ .header("Content-Type", mediaType)
+ .PUT(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+ if (resp.statusCode() >= 400) {
+ throw registryError("PUT manifest " + name + "/" + reference, resp);
+ }
+ return resp.headers().firstValue("Docker-Content-Digest").orElse(null);
+ }
+
+ /** HEAD a blob, returning its size in bytes, or {@code null} if the repository does not hold it. */
+ public Long headBlob(String name, String digest) throws IOException, InterruptedException {
+ HttpResponse<Void> resp = http.send(
+ HttpRequest.newBuilder(URI.create(baseUrl + "/v2/" + name + "/blobs/" + digest))
+ .timeout(Duration.ofSeconds(10))
+ .method("HEAD", HttpRequest.BodyPublishers.noBody())
+ .build(),
+ HttpResponse.BodyHandlers.discarding());
+ if (resp.statusCode() == 404) {
+ return null;
+ }
+ if (resp.statusCode() >= 400) {
+ throw new IOException("Registry HEAD blob " + name + "/" + digest + " returned " + resp.statusCode());
+ }
+ return resp.headers().firstValueAsLong("Content-Length").orElse(0L);
+ }
+
+ /** Opens a blob upload session, returning the absolute URL the next request goes to. */
+ public String startUpload(String name) throws IOException, InterruptedException {
+ HttpResponse<String> resp = http.send(
+ HttpRequest.newBuilder(URI.create(baseUrl + "/v2/" + name + "/blobs/uploads/"))
+ .timeout(Duration.ofSeconds(10))
+ .POST(HttpRequest.BodyPublishers.noBody())
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+ if (resp.statusCode() != 202) {
+ throw registryError("POST blob upload " + name, resp);
+ }
+ return uploadLocation(resp);
+ }
+
+ /**
+ * Sends one chunk of an upload, bytes {@code first} to {@code first + bytes.length - 1}, and
+ * returns the URL the next request goes to.
+ */
+ public String patchUpload(String location, byte[] bytes, long first) throws IOException, InterruptedException {
+ HttpResponse<String> resp = http.send(
+ HttpRequest.newBuilder(URI.create(location))
+ .timeout(Duration.ofSeconds(60))
+ .header("Content-Type", "application/octet-stream")
+ .header("Content-Range", first + "-" + (first + bytes.length - 1))
+ .method("PATCH", HttpRequest.BodyPublishers.ofByteArray(bytes))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+ if (resp.statusCode() != 202) {
+ throw registryError("PATCH blob upload", resp);
+ }
+ return uploadLocation(resp);
+ }
+
+ /** Closes an upload, asking the registry to store what it received under {@code digest}. */
+ public void finishUpload(String location, String digest) throws IOException, InterruptedException {
+ String url = location + (location.contains("?") ? "&" : "?")
+ + "digest=" + URLEncoder.encode(digest, StandardCharsets.UTF_8);
+ HttpResponse<String> resp = http.send(
+ HttpRequest.newBuilder(URI.create(url))
+ .timeout(Duration.ofSeconds(60))
+ .header("Content-Type", "application/octet-stream")
+ .PUT(HttpRequest.BodyPublishers.noBody())
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+ if (resp.statusCode() != 201) {
+ throw registryError("PUT blob upload", resp);
+ }
+ }
+
+ /** Drops an upload session the registry still holds. Best effort. */
+ public void cancelUpload(String location) {
+ try {
+ http.send(HttpRequest.newBuilder(URI.create(location))
+ .timeout(Duration.ofSeconds(10))
+ .DELETE()
+ .build(),
+ HttpResponse.BodyHandlers.discarding());
+ } catch (Exception e) {
+ LOG.debugv("Registry DELETE blob upload failed: {0}", e.getMessage());
+ }
+ }
+
+ private String uploadLocation(HttpResponse<?> resp) throws IOException {
+ String location = resp.headers().firstValue("Location").orElse(null);
+ if (location == null || location.isBlank()) {
+ throw new IOException("Registry blob upload response has no Location");
+ }
+ return location.startsWith("/") ? baseUrl + location : location;
+ }
+
+ private static RegistryErrorException registryError(String what, HttpResponse<String> resp) {
+ String code = null;
+ try {
+ JsonNode errors = MAPPER.readTree(resp.body()).path("errors");
+ if (errors.isArray() && !errors.isEmpty()) {
+ code = errors.get(0).path("code").asText(null);
+ }
+ } catch (Exception ignored) {
+ // A body that is not the registry's error JSON leaves the code unknown.
+ }
+ return new RegistryErrorException(resp.statusCode(), code,
+ "Registry " + what + " returned " + resp.statusCode() + ": " + resp.body());
+ }
+
/**
* Sums {@code config.size} + each {@code layers[].size} from a manifest body.
* Returns 0 if the body is not a recognised image manifest.
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecr/EcrImagePushApiDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecr/EcrImagePushApiDockerIntegrationTest.java
new file mode 100644
index 0000000000..ed19cb3b55
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecr/EcrImagePushApiDockerIntegrationTest.java
@@ -0,0 +1,304 @@
+package io.github.hectorvent.floci.services.ecr;
+
+import io.github.hectorvent.floci.core.common.Digests;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.github.hectorvent.floci.testing.SidecarContainersProfile;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.TestProfile;
+import io.restassured.RestAssured;
+import io.restassured.response.ValidatableResponse;
+import io.vertx.core.Vertx;
+import io.vertx.core.buffer.Buffer;
+import io.vertx.core.http.HttpClient;
+import io.vertx.core.http.HttpMethod;
+import io.vertx.core.http.RequestOptions;
+import io.vertx.core.json.Json;
+import io.vertx.core.net.SocketAddress;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.hasSize;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Pushes an image through the ECR API alone, as SDK-based copy tools do: a layer uploaded in parts,
+ * a config blob, then {@code PutImage}. The result must be the image the registry data plane, and
+ * so {@code docker pull}, serves, and a retag through {@code PutImage} must reach it too.
+ */
+@QuarkusTest
+@TestProfile(SidecarContainersProfile.class)
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+class EcrImagePushApiDockerIntegrationTest {
+
+ private static final String CT = "application/x-amz-json-1.1";
+ private static final String PREFIX = "AmazonEC2ContainerRegistry_V20150921.";
+ private static final int MIB = 1024 * 1024;
+ private static final String DOCKER_MANIFEST = "application/vnd.docker.distribution.manifest.v2+json";
+
+ private final String repo = "push-api/" + UUID.randomUUID().toString().substring(0, 8);
+ private final String immutableRepo = repo + "-immutable";
+ private final byte[] layer = randomBytes(5 * MIB + 1000);
+ private final byte[] config = "{\"architecture\":\"amd64\",\"os\":\"linux\"}".getBytes(StandardCharsets.UTF_8);
+ private final String layerDigest = "sha256:" + Digests.sha256Hex(layer);
+ private final String configDigest = "sha256:" + Digests.sha256Hex(config);
+ private String manifest;
+ private String manifestDigest;
+
+ @Inject
+ Vertx vertx;
+
+ private boolean created;
+
+ @BeforeAll
+ static void requireDocker() {
+ SidecarContainersProfile.requireDockerAndImage("floci.services.ecr.registry-image");
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ /** Once, but per test: RestAssured has its port only once a test starts. */
+ @BeforeEach
+ void setUp() {
+ if (created) {
+ return;
+ }
+ created = true;
+ call("CreateRepository", "{\"repositoryName\":\"" + repo + "\"}").statusCode(200);
+ call("CreateRepository", "{\"repositoryName\":\"" + immutableRepo + "\",\"imageTagMutability\":\"IMMUTABLE\"}")
+ .statusCode(200);
+ manifest = """
+ {
+ "schemaVersion": 2,
+ "mediaType": "%s",
+ "config": {"mediaType": "application/vnd.docker.container.image.v1+json", "size": %d, "digest": "%s"},
+ "layers": [{"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", "size": %d, "digest": "%s"}]
+ }""".formatted(DOCKER_MANIFEST, config.length, configDigest, layer.length, layerDigest);
+ manifestDigest = "sha256:" + Digests.sha256Hex(manifest.getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Test
+ @Order(1)
+ void aLayerUploadsInPartsThatMustFollowEachOther() {
+ String uploadId = initiate(repo);
+ byte[] first = Arrays.copyOfRange(layer, 0, 5 * MIB);
+ byte[] rest = Arrays.copyOfRange(layer, 5 * MIB, layer.length);
+
+ part(repo, uploadId, 0, first).statusCode(200).body("lastByteReceived", equalTo(5 * MIB - 1));
+ part(repo, uploadId, 0, first).statusCode(400)
+ .body("__type", equalTo("InvalidLayerPartException"))
+ .body("lastValidByteReceived", equalTo(5 * MIB - 1));
+ part(repo, uploadId, 5 * MIB, rest).statusCode(200).body("lastByteReceived", equalTo(layer.length - 1));
+
+ complete(repo, uploadId, layerDigest).statusCode(200)
+ .body("layerDigest", equalTo(layerDigest))
+ .body("uploadId", equalTo(uploadId));
+ complete(repo, uploadId, layerDigest).statusCode(400).body("__type", equalTo("UploadNotFoundException"));
+
+ String configUpload = initiate(repo);
+ part(repo, configUpload, 0, config).statusCode(200);
+ complete(repo, configUpload, configDigest).statusCode(200);
+ }
+
+ @Test
+ @Order(2)
+ void aBadUploadIsRefused() {
+ String again = initiate(repo);
+ part(repo, again, 0, config).statusCode(200);
+ complete(repo, again, configDigest).statusCode(400).body("__type", equalTo("LayerAlreadyExistsException"));
+
+ String empty = initiate(repo);
+ complete(repo, empty, layerDigest).statusCode(400).body("__type", equalTo("EmptyUploadException"));
+
+ String small = initiate(repo);
+ part(repo, small, 0, new byte[10]).statusCode(200);
+ part(repo, small, 10, new byte[10]).statusCode(200);
+ complete(repo, small, "sha256:" + Digests.sha256Hex(new byte[20])).statusCode(400)
+ .body("__type", equalTo("LayerPartTooSmallException"));
+
+ String mismatch = initiate(repo);
+ part(repo, mismatch, 0, "other".getBytes(StandardCharsets.UTF_8)).statusCode(200);
+ complete(repo, mismatch, "sha256:" + Digests.sha256Hex(new byte[1])).statusCode(400)
+ .body("__type", equalTo("InvalidLayerException"));
+ }
+
+ @Test
+ @Order(3)
+ void uploadedLayersAreAvailable() {
+ String missing = "sha256:" + "0".repeat(64);
+ call("BatchCheckLayerAvailability", "{\"repositoryName\":\"" + repo + "\",\"layerDigests\":[\""
+ + layerDigest + "\",\"" + configDigest + "\",\"" + missing + "\",\"bad\"]}")
+ .statusCode(200)
+ .body("layers", hasSize(2))
+ .body("layers[0].layerDigest", equalTo(layerDigest))
+ .body("layers[0].layerAvailability", equalTo("AVAILABLE"))
+ .body("layers[0].layerSize", equalTo(layer.length))
+ .body("layers[1].layerSize", equalTo(config.length))
+ .body("failures.failureCode", hasItems("MissingLayerDigest", "InvalidLayerDigest"))
+ .body("failures.layerDigest", hasItems(missing, "bad"));
+ }
+
+ @Test
+ @Order(4)
+ void putImageWritesTheManifestThatPullsServe() throws Exception {
+ call("PutImage", putImage(repo, manifest, "v1")).statusCode(200)
+ .body("image.imageId.imageTag", equalTo("v1"))
+ .body("image.imageId.imageDigest", equalTo(manifestDigest))
+ .body("image.imageManifestMediaType", equalTo(DOCKER_MANIFEST))
+ .body("image.repositoryName", equalTo(repo));
+ call("PutImage", putImage(repo, manifest, "v1")).statusCode(400)
+ .body("__type", equalTo("ImageAlreadyExistsException"));
+
+ String unknownLayer = manifest.replace(layerDigest, "sha256:" + "1".repeat(64));
+ call("PutImage", putImage(repo, unknownLayer, "broken")).statusCode(400)
+ .body("__type", equalTo("LayersNotFoundException"));
+
+ Fetched pulled = registryGet("000000000000.dkr.ecr.us-east-1.localhost",
+ "/v2/" + repo + "/manifests/v1", DOCKER_MANIFEST);
+ assertEquals(200, pulled.statusCode());
+ assertEquals(manifestDigest, pulled.digest());
+ }
+
+ @Test
+ @Order(5)
+ void aRetagReadsTheManifestAndPutsItUnderANewTag() {
+ String read = call("BatchGetImage", "{\"repositoryName\":\"" + repo + "\",\"imageIds\":[{\"imageTag\":\"v1\"}]}")
+ .statusCode(200)
+ .extract().path("images[0].imageManifest");
+
+ call("PutImage", putImage(repo, read, "release")).statusCode(200)
+ .body("image.imageId.imageDigest", equalTo(manifestDigest));
+ call("DescribeImages", "{\"repositoryName\":\"" + repo + "\",\"imageIds\":[{\"imageTag\":\"release\"}]}")
+ .statusCode(200)
+ .body("imageDetails[0].imageDigest", equalTo(manifestDigest));
+ }
+
+ @Test
+ @Order(6)
+ void theDownloadUrlServesTheLayer() throws Exception {
+ String url = call("GetDownloadUrlForLayer", "{\"repositoryName\":\"" + repo + "\",\"layerDigest\":\""
+ + layerDigest + "\"}")
+ .statusCode(200)
+ .body("layerDigest", equalTo(layerDigest))
+ .extract().path("downloadUrl");
+ URI uri = URI.create(url);
+ assertEquals("/v2/" + repo + "/blobs/" + layerDigest, uri.getPath());
+
+ Fetched response = registryGet(uri.getHost(), uri.getPath(), null);
+ assertEquals(200, response.statusCode());
+ assertArrayEquals(layer, response.body().getBytes());
+
+ call("GetDownloadUrlForLayer", "{\"repositoryName\":\"" + repo + "\",\"layerDigest\":\"sha256:"
+ + "2".repeat(64) + "\"}")
+ .statusCode(400).body("__type", equalTo("LayersNotFoundException"));
+ }
+
+ @Test
+ @Order(7)
+ void anImmutableTagIsWrittenOnce() {
+ String upload = initiate(immutableRepo);
+ part(immutableRepo, upload, 0, config).statusCode(200);
+ complete(immutableRepo, upload, configDigest).statusCode(200);
+ String first = """
+ {"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",\
+ "config":{"mediaType":"application/vnd.oci.image.config.v1+json","size":%d,"digest":"%s"},\
+ "layers":[]}""".formatted(config.length, configDigest);
+ String second = first.replace("\"layers\":[]", "\"layers\":[],\"annotations\":{\"build\":\"2\"}");
+
+ call("PutImage", putImage(immutableRepo, first, "v1")).statusCode(200);
+ call("PutImage", putImage(immutableRepo, second, "v1")).statusCode(400)
+ .body("__type", equalTo("ImageTagAlreadyExistsException"));
+ call("PutImage", putImage(immutableRepo, second, "v2")).statusCode(200)
+ .body("image.imageManifestMediaType", equalTo("application/vnd.oci.image.manifest.v1+json"));
+ }
+
+ /** Last, since every class on this profile shares one registry. */
+ @Test
+ @Order(8)
+ void cleanUp() {
+ call("DeleteRepository", "{\"repositoryName\":\"" + repo + "\",\"force\":true}").statusCode(200);
+ call("DeleteRepository", "{\"repositoryName\":\"" + immutableRepo + "\",\"force\":true}").statusCode(200);
+ }
+
+ private String initiate(String repository) {
+ return call("InitiateLayerUpload", "{\"repositoryName\":\"" + repository + "\"}")
+ .statusCode(200)
+ .body("partSize", equalTo(10 * MIB))
+ .extract().path("uploadId");
+ }
+
+ private static ValidatableResponse part(String repository, String uploadId, long first, byte[] bytes) {
+ return call("UploadLayerPart", "{\"repositoryName\":\"" + repository + "\",\"uploadId\":\"" + uploadId
+ + "\",\"partFirstByte\":" + first + ",\"partLastByte\":" + (first + bytes.length - 1)
+ + ",\"layerPartBlob\":\"" + Base64.getEncoder().encodeToString(bytes) + "\"}");
+ }
+
+ private static ValidatableResponse complete(String repository, String uploadId, String digest) {
+ return call("CompleteLayerUpload", "{\"repositoryName\":\"" + repository + "\",\"uploadId\":\"" + uploadId
+ + "\",\"layerDigests\":[\"" + digest + "\"]}");
+ }
+
+ private static String putImage(String repository, String imageManifest, String tag) {
+ return "{\"repositoryName\":\"" + repository + "\",\"imageTag\":\"" + tag + "\",\"imageManifest\":"
+ + Json.encode(imageManifest) + "}";
+ }
+
+ /** A GET through Floci's registry data plane, by the host name a Docker client would use. */
+ private Fetched registryGet(String host, String path, String accept) throws Exception {
+ int port = RestAssured.port;
+ HttpClient client = vertx.createHttpClient();
+ RequestOptions options = new RequestOptions()
+ .setServer(SocketAddress.inetSocketAddress(port, "127.0.0.1"))
+ .setHost(host)
+ .setPort(port)
+ .setMethod(HttpMethod.GET)
+ .setURI(path);
+ return client.request(options)
+ .compose(request -> {
+ if (accept != null) {
+ request.putHeader("Accept", accept);
+ }
+ return request.send();
+ })
+ .compose(response -> response.body().map(body -> new Fetched(
+ response.statusCode(), response.getHeader("Docker-Content-Digest"), body)))
+ .onComplete(ignored -> client.close())
+ .toCompletionStage().toCompletableFuture().get(30, TimeUnit.SECONDS);
+ }
+
+ /** A data plane response: its status, its {@code Docker-Content-Digest} and its body. */
+ private record Fetched(int statusCode, String digest, Buffer body) {}
+
+ private static ValidatableResponse call(String action, String body) {
+ return given()
+ .header("X-Amz-Target", PREFIX + action)
+ .contentType(CT)
+ .body(body)
+ .when()
+ .post("/")
+ .then();
+ }
+
+ private static byte[] randomBytes(int size) {
+ byte[] bytes = new byte[size];
+ new Random(5735).nextBytes(bytes);
+ return bytes;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecr/EcrImagePushApiIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecr/EcrImagePushApiIntegrationTest.java
new file mode 100644
index 0000000000..1c2eb72c09
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecr/EcrImagePushApiIntegrationTest.java
@@ -0,0 +1,161 @@
+package io.github.hectorvent.floci.services.ecr;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.ValidatableResponse;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * {@code PutImage} and the layer API's checks that come before the registry is reached: unknown
+ * repositories, unknown uploads and malformed input. Needs no Docker; the push itself is covered
+ * by {@code EcrImagePushApiDockerIntegrationTest}.
+ */
+@QuarkusTest
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+class EcrImagePushApiIntegrationTest {
+
+ private static final String CT = "application/x-amz-json-1.1";
+ private static final String PREFIX = "AmazonEC2ContainerRegistry_V20150921.";
+ private static final String REPO = "push-api-it/checks";
+ private static final String OCI_MANIFEST = "application/vnd.oci.image.manifest.v1+json";
+ private static final String DIGEST = "sha256:" + "a".repeat(64);
+ private static final String UPLOAD_ID = "6a1f2b3c-0000-4000-8000-000000000000";
+
+ private boolean created;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ /** Once, but per test: RestAssured has its port only once a test starts. */
+ @BeforeEach
+ void createRepository() {
+ if (!created) {
+ call("CreateRepository", "{\"repositoryName\":\"" + REPO + "\"}").statusCode(200);
+ created = true;
+ }
+ }
+
+ @Test
+ void everyActionNeedsAnExistingRepository() {
+ String missing = "\"repositoryName\":\"push-api-it/missing\"";
+ String[][] calls = {
+ {"PutImage", "{" + missing + ",\"imageManifest\":\"{}\",\"imageTag\":\"v1\"}"},
+ {"BatchCheckLayerAvailability", "{" + missing + ",\"layerDigests\":[\"" + DIGEST + "\"]}"},
+ {"GetDownloadUrlForLayer", "{" + missing + ",\"layerDigest\":\"" + DIGEST + "\"}"},
+ {"InitiateLayerUpload", "{" + missing + "}"},
+ {"UploadLayerPart", "{" + missing + ",\"uploadId\":\"" + UPLOAD_ID
+ + "\",\"partFirstByte\":0,\"partLastByte\":0,\"layerPartBlob\":\"YQ==\"}"},
+ {"CompleteLayerUpload", "{" + missing + ",\"uploadId\":\"" + UPLOAD_ID
+ + "\",\"layerDigests\":[\"" + DIGEST + "\"]}"},
+ };
+ for (String[] c : calls) {
+ call(c[0], c[1]).statusCode(400).body("__type", equalTo("RepositoryNotFoundException"));
+ }
+ }
+
+ @Test
+ void putImageChecksTheManifestAndItsDigest() {
+ call("PutImage", "{\"repositoryName\":\"" + REPO + "\",\"imageManifest\":\"{}\",\"imageTag\":\"v1\","
+ + "\"imageManifestMediaType\":\"" + OCI_MANIFEST + "\",\"imageDigest\":\"" + DIGEST + "\"}")
+ .statusCode(400)
+ .body("__type", equalTo("ImageDigestDoesNotMatchException"))
+ .body("message", containsString("sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"));
+ call("PutImage", "{\"repositoryName\":\"" + REPO + "\",\"imageManifest\":\"not json\",\"imageTag\":\"v1\"}")
+ .statusCode(400).body("__type", equalTo("InvalidParameterException"));
+ call("PutImage", "{\"repositoryName\":\"" + REPO + "\",\"imageManifest\":\"\",\"imageTag\":\"v1\"}")
+ .statusCode(400).body("__type", equalTo("InvalidParameterException"));
+ // With no mediaType in the manifest, AWS requires imageManifestMediaType rather than guessing.
+ call("PutImage", "{\"repositoryName\":\"" + REPO + "\",\"imageManifest\":\"{}\",\"imageTag\":\"v1\"}")
+ .statusCode(400)
+ .body("__type", equalTo("InvalidParameterException"))
+ .body("message", containsString("imageManifestMediaType"));
+ }
+
+ @Test
+ void putImageRefusesATagEcrRefusesWithEcrsMessage() {
+ // Each would otherwise change the registry URL: v1?x writes v1, a slash names another repository.
+ for (String tag : new String[] {"v1?x", "team/v1", "v1#x", "v 1", "v1:x"}) {
+ putTag(tag).statusCode(400)
+ .body("__type", equalTo("InvalidParameterException"))
+ .body("message", equalTo("Invalid parameter at 'imageTag' failed to satisfy constraint: "
+ + "'must satisfy regular expression '[a-zA-Z0-9-_.]+''"));
+ }
+ putTag("t".repeat(301)).statusCode(400)
+ .body("message", containsString("Member must have length less than or equal to 300"));
+ }
+
+ @Test
+ void putImageRefusesATagTheBackingRegistryCannotHold() {
+ // ECR takes these; the OCI tag grammar the registry follows does not.
+ for (String tag : new String[] {".v1", "-v1", "t".repeat(129), "t".repeat(300)}) {
+ putTag(tag).statusCode(400)
+ .body("__type", equalTo("InvalidParameterException"))
+ .body("message", containsString("Floci's backing registry cannot hold the tag"));
+ }
+ }
+
+ private static ValidatableResponse putTag(String tag) {
+ return call("PutImage", "{\"repositoryName\":\"" + REPO + "\",\"imageManifest\":\"{}\","
+ + "\"imageManifestMediaType\":\"" + OCI_MANIFEST + "\",\"imageTag\":\"" + tag + "\"}");
+ }
+
+ @Test
+ void layerDigestsAreChecked() {
+ call("GetDownloadUrlForLayer", "{\"repositoryName\":\"" + REPO + "\",\"layerDigest\":\"not-a-digest\"}")
+ .statusCode(400).body("__type", equalTo("InvalidParameterException"));
+ call("BatchCheckLayerAvailability", "{\"repositoryName\":\"" + REPO + "\",\"layerDigests\":[]}")
+ .statusCode(400).body("__type", equalTo("InvalidParameterException"));
+ call("CompleteLayerUpload", "{\"repositoryName\":\"" + REPO + "\",\"uploadId\":\"" + UPLOAD_ID
+ + "\",\"layerDigests\":[]}")
+ .statusCode(400).body("__type", equalTo("InvalidParameterException"));
+ call("CompleteLayerUpload", "{\"repositoryName\":\"" + REPO + "\",\"uploadId\":\"" + UPLOAD_ID
+ + "\",\"layerDigests\":[\"bad\"]}")
+ .statusCode(400).body("__type", equalTo("InvalidParameterException"));
+ // An upload holds one layer: a second, different digest is refused, not ignored.
+ call("CompleteLayerUpload", "{\"repositoryName\":\"" + REPO + "\",\"uploadId\":\"" + UPLOAD_ID
+ + "\",\"layerDigests\":[\"sha256:" + "a".repeat(64) + "\",\"sha256:" + "b".repeat(64) + "\"]}")
+ .statusCode(400).body("__type", equalTo("InvalidParameterException"));
+ }
+
+ @Test
+ void anUnknownUploadIsNotFound() {
+ call("UploadLayerPart", "{\"repositoryName\":\"" + REPO + "\",\"uploadId\":\"" + UPLOAD_ID
+ + "\",\"partFirstByte\":0,\"partLastByte\":0,\"layerPartBlob\":\"YQ==\"}")
+ .statusCode(400).body("__type", equalTo("UploadNotFoundException"));
+ call("CompleteLayerUpload", "{\"repositoryName\":\"" + REPO + "\",\"uploadId\":\"" + UPLOAD_ID
+ + "\",\"layerDigests\":[\"" + DIGEST + "\"]}")
+ .statusCode(400).body("__type", equalTo("UploadNotFoundException"));
+ }
+
+ @Test
+ void uploadPartInputIsChecked() {
+ call("UploadLayerPart", "{\"repositoryName\":\"" + REPO + "\",\"uploadId\":\"" + UPLOAD_ID
+ + "\",\"partFirstByte\":-1,\"partLastByte\":0,\"layerPartBlob\":\"YQ==\"}")
+ .statusCode(400).body("__type", equalTo("InvalidParameterException"));
+ call("UploadLayerPart", "{\"repositoryName\":\"" + REPO + "\",\"uploadId\":\"" + UPLOAD_ID
+ + "\",\"partFirstByte\":0,\"partLastByte\":0,\"layerPartBlob\":\"%%%\"}")
+ .statusCode(400).body("__type", equalTo("InvalidParameterException"));
+ call("UploadLayerPart", "{\"repositoryName\":\"" + REPO + "\",\"uploadId\":\"" + UPLOAD_ID
+ + "\",\"partFirstByte\":\"zero\",\"partLastByte\":0,\"layerPartBlob\":\"YQ==\"}")
+ .statusCode(400).body("__type", equalTo("InvalidParameterException"));
+ }
+
+ private static ValidatableResponse call(String action, String body) {
+ return given()
+ .header("X-Amz-Target", PREFIX + action)
+ .contentType(CT)
+ .body(body)
+ .when()
+ .post("/")
+ .then();
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecr/EcrLayerUploadsTest.java b/src/test/java/io/github/hectorvent/floci/services/ecr/EcrLayerUploadsTest.java
new file mode 100644
index 0000000000..65e237dc87
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecr/EcrLayerUploadsTest.java
@@ -0,0 +1,159 @@
+package io.github.hectorvent.floci.services.ecr;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import org.junit.jupiter.api.Test;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class EcrLayerUploadsTest {
+
+ private static final String SCOPE = "us-east-1::000000000000::app";
+ private static final long MIB = 1024 * 1024;
+
+ @Test
+ void aPartMustStartAtTheNextByteAndMatchItsRange() {
+ EcrLayerUploads uploads = new EcrLayerUploads(location -> { });
+ EcrLayerUploads.Session session = uploads.require(SCOPE, uploads.open(SCOPE, "http://r/u1"), "app");
+
+ assertDoesNotThrow(() -> check(session, 0, 9, 10));
+ session.accept(10, "http://r/u2", Instant.now());
+
+ AwsException gap = assertThrows(AwsException.class, () -> check(session, 11, 20, 10));
+ assertEquals("InvalidLayerPartException", gap.getErrorCode());
+ assertEquals(9L, gap.getExtendedData().get("lastValidByteReceived"));
+ assertEquals("app", gap.getExtendedData().get("repositoryName"));
+
+ assertEquals("InvalidLayerPartException",
+ assertThrows(AwsException.class, () -> check(session, 0, 9, 10)).getErrorCode());
+ assertEquals("InvalidLayerPartException",
+ assertThrows(AwsException.class, () -> check(session, 10, 19, 9)).getErrorCode());
+ assertEquals("InvalidLayerPartException",
+ assertThrows(AwsException.class, () -> check(session, 10, 9, 0)).getErrorCode());
+ assertDoesNotThrow(() -> check(session, 10, 19, 10));
+ }
+
+ @Test
+ void aPartOver20MiBIsRefused() {
+ EcrLayerUploads uploads = new EcrLayerUploads(location -> { });
+ EcrLayerUploads.Session session = uploads.require(SCOPE, uploads.open(SCOPE, "http://r/u"), "app");
+ int tooBig = (int) (EcrLayerUploads.MAX_PART_SIZE + 1);
+
+ assertEquals("InvalidLayerPartException",
+ assertThrows(AwsException.class, () -> check(session, 0, tooBig - 1, tooBig)).getErrorCode());
+ int largest = (int) EcrLayerUploads.MAX_PART_SIZE;
+ assertDoesNotThrow(() -> check(session, 0, largest - 1, largest));
+ }
+
+ @Test
+ void completionNeedsAPartAndEveryPartButTheLastAtLeast5MiB() {
+ EcrLayerUploads uploads = new EcrLayerUploads(location -> { });
+ EcrLayerUploads.Session empty = uploads.require(SCOPE, uploads.open(SCOPE, "http://r/a"), "app");
+ assertEquals("EmptyUploadException",
+ assertThrows(AwsException.class, () -> EcrLayerUploads.checkComplete(empty)).getErrorCode());
+
+ EcrLayerUploads.Session small = uploads.require(SCOPE, uploads.open(SCOPE, "http://r/b"), "app");
+ small.accept(5 * MIB - 1, "http://r/b2", Instant.now());
+ small.accept(10, "http://r/b3", Instant.now());
+ assertEquals("LayerPartTooSmallException",
+ assertThrows(AwsException.class, () -> EcrLayerUploads.checkComplete(small)).getErrorCode());
+
+ EcrLayerUploads.Session ok = uploads.require(SCOPE, uploads.open(SCOPE, "http://r/c"), "app");
+ ok.accept(5 * MIB, "http://r/c2", Instant.now());
+ ok.accept(10, "http://r/c3", Instant.now());
+ assertDoesNotThrow(() -> EcrLayerUploads.checkComplete(ok));
+
+ EcrLayerUploads.Session single = uploads.require(SCOPE, uploads.open(SCOPE, "http://r/d"), "app");
+ single.accept(10, "http://r/d2", Instant.now());
+ assertDoesNotThrow(() -> EcrLayerUploads.checkComplete(single));
+ }
+
+ @Test
+ void anUploadBelongsToItsRepositoryAndEndsWhenClosed() {
+ EcrLayerUploads uploads = new EcrLayerUploads(location -> { });
+ String id = uploads.open(SCOPE, "http://r/u");
+
+ assertEquals("UploadNotFoundException", assertThrows(AwsException.class,
+ () -> uploads.require("us-east-1::000000000000::other", id, "other")).getErrorCode());
+ assertEquals("UploadNotFoundException", assertThrows(AwsException.class,
+ () -> uploads.require(SCOPE, "6a1f2b3c-0000-4000-8000-000000000000", "app")).getErrorCode());
+ assertEquals("http://r/u", uploads.require(SCOPE, id, "app").location());
+
+ uploads.close(id);
+ assertEquals("UploadNotFoundException",
+ assertThrows(AwsException.class, () -> uploads.require(SCOPE, id, "app")).getErrorCode());
+ }
+
+ @Test
+ void anUploadIdleForADayIsDroppedWhenTheNextOneOpens() {
+ MutableClock clock = new MutableClock(Instant.parse("2026-10-15T00:00:00Z"));
+ List<String> discarded = new ArrayList<>();
+ EcrLayerUploads uploads = new EcrLayerUploads(clock, discarded::add);
+ String stale = uploads.open(SCOPE, "http://r/stale");
+ String touched = uploads.open(SCOPE, "http://r/touched");
+
+ clock.advance(Duration.ofHours(23));
+ uploads.require(SCOPE, touched, "app").accept(10, "http://r/touched2", clock.instant());
+ clock.advance(Duration.ofHours(2));
+ uploads.open(SCOPE, "http://r/new");
+
+ assertEquals("UploadNotFoundException",
+ assertThrows(AwsException.class, () -> uploads.require(SCOPE, stale, "app")).getErrorCode());
+ assertEquals("http://r/touched2", uploads.require(SCOPE, touched, "app").location());
+ assertEquals(List.of("http://r/stale"), discarded);
+ }
+
+ @Test
+ void anUploadIdleForADayIsRefusedEvenWhenNoOtherOpens() {
+ MutableClock clock = new MutableClock(Instant.parse("2026-10-15T00:00:00Z"));
+ List<String> discarded = new ArrayList<>();
+ EcrLayerUploads uploads = new EcrLayerUploads(clock, discarded::add);
+ String id = uploads.open(SCOPE, "http://r/idle");
+
+ clock.advance(Duration.ofHours(25));
+
+ assertEquals("UploadNotFoundException",
+ assertThrows(AwsException.class, () -> uploads.require(SCOPE, id, "app")).getErrorCode());
+ assertEquals(List.of("http://r/idle"), discarded);
+ }
+
+ private static void check(EcrLayerUploads.Session session, long first, long last, int length) {
+ EcrLayerUploads.checkPart(session, first, last, length, "000000000000", "app", "upload");
+ }
+
+ private static final class MutableClock extends Clock {
+ private Instant now;
+
+ MutableClock(Instant now) {
+ this.now = now;
+ }
+
+ void advance(Duration duration) {
+ now = now.plus(duration);
+ }
+
+ @Override
+ public Instant instant() {
+ return now;
+ }
+
+ @Override
+ public ZoneOffset getZone() {
+ return ZoneOffset.UTC;
+ }
+
+ @Override
+ public Clock withZone(ZoneId zone) {
+ return this;
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecr/EcrServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ecr/EcrServiceTest.java
index 50c4e96850..17ce9e379d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecr/EcrServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecr/EcrServiceTest.java
@@ -902,6 +902,23 @@ class EcrServiceTest {
assertEquals(1, service.describeRepositories(List.of(repositoryName), null, REGION).size());
}
+ @Test
+ void putImage_retagKeepsTheFirstPushTime() {
+ InMemoryStorage<String, ImageMetadata> metaStore = new InMemoryStorage<>();
+ EcrService withMeta = new EcrService(new InMemoryStorage<>(), metaStore, pullThroughCacheRuleStore,
+ registryManager, Mockito.mock(EmulatorConfig.class), new RegionResolver(REGION, ACCOUNT));
+ when(registryManager.httpClient()).thenReturn(Mockito.mock(RegistryHttpClient.class));
+ withMeta.createRepository(REPO, null, null, null, null, null, null, REGION);
+ String manifest = "{\"schemaVersion\":2,\"mediaType\":\"application/vnd.docker.distribution.manifest.v2+json\"}";
+
+ withMeta.putImage(REPO, manifest, null, "v1", null, null, REGION);
+ Instant firstPush = metaStore.scan(k -> true).get(0).getPushedAt();
+ withMeta.putImage(REPO, manifest, null, "release", null, null, REGION);
+
+ assertEquals(1, metaStore.scan(k -> true).size());
+ assertEquals(firstPush, metaStore.scan(k -> true).get(0).getPushedAt());
+ }
+
@Test
void putImageTagMutability_roundTrips() {
service.createRepository(REPO, null, null, null, null, null, null, REGION);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlaneTest.java b/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlaneTest.java
index 81c622e05f..3ba7a25698 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlaneTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlaneTest.java
@@ -25,6 +25,8 @@ import org.mockito.Mockito;
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -36,6 +38,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
/** Unit tests for routing AWS-shaped ECR repository URIs through Floci's data plane. */
@@ -563,6 +566,11 @@ class EcrRegistryDataPlaneTest {
.thenReturn(new RegistryHttpClient("http://127.0.0.1:" + registry.actualPort()));
when(ecrService.isImageTagImmutable("platform/api", "000000000000", "us-east-1"))
.thenReturn(immutableTags);
+ // The tag lock lives in EcrService, so the mock holds a real one.
+ Set<String> heldTags = ConcurrentHashMap.newKeySet();
+ when(ecrService.claimImmutableTag(anyString())).thenAnswer(call -> heldTags.add(call.getArgument(0)));
+ Mockito.doAnswer(call -> heldTags.remove(call.getArgument(0)))
+ .when(ecrService).releaseImmutableTag(anyString());
when(ecrService.registryRepositoryName("platform/api", "000000000000", "us-east-1"))
.thenReturn("legacy/platform-api");