Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ecr): give every data plane exchange its own registry connection (#5883)
Alessandro Lattaode94e3cfcc03
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlane.java b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlane.javaindex d67b811bdf..2b2b7fdc6a 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlane.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlane.java@@ -24,6 +24,7 @@ import java.nio.charset.StandardCharsets;import java.util.Optional;import java.util.Set;import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.atomic.AtomicBoolean;import java.util.regex.Matcher;import java.util.regex.Pattern;@@ -45,7 +46,6 @@ public class EcrRegistryDataPlane {private final EcrService ecrService;private final EmulatorConfig config;private final Vertx vertx;- private final HttpClient proxyClient;private final Set<String> inFlightImmutableTags = ConcurrentHashMap.newKeySet();@Inject@@ -57,9 +57,6 @@ public class EcrRegistryDataPlane {this.ecrService = ecrService;this.config = config;this.vertx = vertx;- this.proxyClient = vertx.createHttpClient(new HttpClientOptions()- .setConnectTimeout(5_000)- .setKeepAlive(true));}void register(@Observes Router router) {@@ -91,7 +88,14 @@ public class EcrRegistryDataPlane {return;}- checkTagAndProxy(context, registryRequest, immutableTag);+ Exchange exchange = new Exchange(immutableTag);+ // A client that goes away, by resetting its HTTP/2 stream or closing the connection, ends+ // the exchange at any point, the registry check included: otherwise the registry keeps+ // waiting for the rest of a body with a Content-Length, and the exchange's connection,+ // client and immutable tag lock stay.+ context.response().closeHandler(closed -> exchange.end());+ context.request().exceptionHandler(failure -> exchange.end());+ checkTagAndProxy(context, registryRequest, exchange);}private String immutableTag(RoutingContext context, RegistryRequest request) {@@ -102,7 +106,8 @@ public class EcrRegistryDataPlane {return request.accountId() + "/" + request.region() + "/" + request.repositoryName() + ":" + request.tag();}- private void checkTagAndProxy(RoutingContext context, RegistryRequest request, String immutableTag) {+ private void checkTagAndProxy(RoutingContext context, RegistryRequest request, Exchange exchange) {+ String immutableTag = exchange.immutableTag;vertx.<Boolean>executeBlocking(promise -> {try {registryManager.ensureStarted();@@ -118,57 +123,72 @@ public class EcrRegistryDataPlane {promise.fail(e);}}).onComplete(result -> {+ if (exchange.ended()) {+ // The client went away during the check: nothing of this exchange reaches the+ // registry.+ return;+ }if (result.failed()) {context.request().resume();- release(immutableTag);+ exchange.end();context.response().setStatusCode(503).end();} else if (result.result()) {context.request().resume();- release(immutableTag);+ exchange.end();imageTagAlreadyExists(context, request);} else {- proxy(context, request, immutableTag);+ proxy(context, request, exchange);}});}- private void proxy(RoutingContext context, RegistryRequest request, String immutableTag) {+ private void proxy(RoutingContext context, RegistryRequest request, Exchange exchange) {URI backend = URI.create(registryManager.httpClient().baseUrl());RequestOptions options = new RequestOptions().setHost(backend.getHost()).setPort(backend.getPort() == -1 ? 80 : backend.getPort()).setURI(request.backendUri()).setMethod(context.request().method());- proxyClient.request(options).onComplete(upstreamRequest -> {+ // A registry connection of its own for every exchange, never a place in a shared pool's+ // queue. An HTTP/2 client such as crane sends a whole push as streams on one connection,+ // so a pool of any size can be full of its uploads while another of its streams waits,+ // paused. The waiting stream's unread bytes hold that connection's flow-control window,+ // the uploads that do have a registry connection stop receiving data, and none of them+ // ever gives its connection back.+ HttpClient client = exchange.open(vertx.createHttpClient(new HttpClientOptions().setConnectTimeout(5_000)));+ client.request(options).onComplete(upstreamRequest -> {if (upstreamRequest.failed()) {context.request().resume();- release(immutableTag);- context.response().setStatusCode(503).end();+ exchange.end();+ failUnlessGone(context, 503);return;}HttpClientRequest clientReq = upstreamRequest.result();copyRequestHeaders(context, clientReq);clientReq.response().onComplete(upstreamResponse -> {if (upstreamResponse.failed()) {- release(immutableTag);- if (!context.response().ended()) {- context.response().setStatusCode(502).end();- }+ exchange.end();+ failUnlessGone(context, 502);return;}copyResponseHeaders(context, upstreamResponse.result(), request);- upstreamResponse.result().pipeTo(context.response()).onComplete(ignored -> release(immutableTag));+ upstreamResponse.result().pipeTo(context.response()).onComplete(ignored -> exchange.end());});clientReq.send(context.request()).onFailure(ignored -> {- release(immutableTag);- if (!context.response().ended()) {- context.response().setStatusCode(502).end();- }+ exchange.end();+ failUnlessGone(context, 502);});context.request().resume();});}+ /** Answers {@code status} unless the response is already written or its client went away. */+ private static void failUnlessGone(RoutingContext context, int status) {+ if (!context.response().ended() && !context.response().closed()) {+ context.response().setStatusCode(status).end();+ }+ }+private static void copyRequestHeaders(RoutingContext context, HttpClientRequest upstream) {context.request().headers().forEach(header -> {if (!HOP_BY_HOP_HEADERS.contains(header.getKey().toLowerCase())@@ -226,6 +246,46 @@ public class EcrRegistryDataPlane {}}+ /**+ * One proxied request: the registry client it opens once it gets that far, and the immutable+ * tag it holds. It ends once, at whichever comes first of the registry's answer, a failure and+ * the client going away; the close handler also fires after an exchange that ended normally,+ * when another exchange may already hold the same tag.+ */+ private final class Exchange {++ private final String immutableTag;+ private final AtomicBoolean ended = new AtomicBoolean();+ private HttpClient client;++ private Exchange(String immutableTag) {+ this.immutableTag = immutableTag;+ }++ private boolean ended() {+ return ended.get();+ }++ private HttpClient open(HttpClient registryClient) {+ client = registryClient;+ return registryClient;+ }++ private void end() {+ if (!ended.compareAndSet(false, true)) {+ return;+ }+ if (client == null) {+ release(immutableTag);+ return;+ }+ // Closing the client closes the exchange's registry connection, which the registry+ // sees and aborts on. The tag is free only once it is closed, so another write of the+ // tag cannot start its check while this one can still reach the registry.+ client.close().onComplete(closed -> release(immutableTag));+ }+ }+static Optional<RegistryRequest> requestFor(String host, String path, String query, String uriStyle) {String registryPath = "/v2".equals(path) ? "/v2/" : path;if (!registryPath.startsWith("/v2/")) {diff --git a/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlaneTest.java b/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlaneTest.javaindex 507366d761..81c622e05f 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlaneTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlaneTest.java@@ -2,21 +2,30 @@ package io.github.hectorvent.floci.services.ecr.registry;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.services.ecr.EcrService;+import io.vertx.core.Future;import io.vertx.core.Vertx;import io.vertx.core.buffer.Buffer;import io.vertx.core.http.HttpClient;+import io.vertx.core.http.HttpClientOptions;+import io.vertx.core.http.HttpClientRequest;+import io.vertx.core.http.HttpClientResponse;import io.vertx.core.http.HttpMethod;import io.vertx.core.http.HttpServer;+import io.vertx.core.http.HttpServerRequest;+import io.vertx.core.http.HttpVersion;import io.vertx.core.http.RequestOptions;import io.vertx.core.json.JsonArray;import io.vertx.core.json.JsonObject;+import io.vertx.core.net.SocketAddress;import io.vertx.ext.web.Router;import org.junit.jupiter.api.AfterEach;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.mockito.Mockito;+import java.util.ArrayList;import java.util.List;+import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicBoolean;import java.util.concurrent.atomic.AtomicInteger;@@ -243,23 +252,7 @@ class EcrRegistryDataPlaneTest {HttpServer dataPlane = null;try {- EcrRegistryManager registryManager = Mockito.mock(EcrRegistryManager.class);- EcrService ecrService = Mockito.mock(EcrService.class);- EmulatorConfig config = Mockito.mock(EmulatorConfig.class, Mockito.RETURNS_DEEP_STUBS);- when(config.services().ecr().enabled()).thenReturn(true);- when(config.services().ecr().uriStyle()).thenReturn("hostname");- when(registryManager.httpClient())- .thenReturn(new RegistryHttpClient("http://127.0.0.1:" + registry.actualPort()));- when(ecrService.isImageTagImmutable("platform/api", "000000000000", "us-east-1"))- .thenReturn(true);- when(ecrService.registryRepositoryName("platform/api", "000000000000", "us-east-1"))- .thenReturn("legacy/platform-api");-- Router router = Router.router(vertx);- new EcrRegistryDataPlane(registryManager, ecrService, config, vertx).register(router);- dataPlane = vertx.createHttpServer().requestHandler(router)- .listen(0, "127.0.0.1")- .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ dataPlane = dataPlane(registry, true);assertEquals(201, putManifest(dataPlane.actualPort()).statusCode());@@ -307,23 +300,7 @@ class EcrRegistryDataPlaneTest {HttpServer dataPlane = null;try {- EcrRegistryManager registryManager = Mockito.mock(EcrRegistryManager.class);- EcrService ecrService = Mockito.mock(EcrService.class);- EmulatorConfig config = Mockito.mock(EmulatorConfig.class, Mockito.RETURNS_DEEP_STUBS);- when(config.services().ecr().enabled()).thenReturn(true);- when(config.services().ecr().uriStyle()).thenReturn("hostname");- when(registryManager.httpClient())- .thenReturn(new RegistryHttpClient("http://127.0.0.1:" + registry.actualPort()));- when(ecrService.isImageTagImmutable("platform/api", "000000000000", "us-east-1"))- .thenReturn(true);- when(ecrService.registryRepositoryName("platform/api", "000000000000", "us-east-1"))- .thenReturn("legacy/platform-api");-- Router router = Router.router(vertx);- new EcrRegistryDataPlane(registryManager, ecrService, config, vertx).register(router);- dataPlane = vertx.createHttpServer().requestHandler(router)- .listen(0, "127.0.0.1")- .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ dataPlane = dataPlane(registry, true);String tlsHost = "000000000000.dkr.ecr.us-east-1.localhost.floci.io:4566";assertEquals(201, putManifest(dataPlane.actualPort(), tlsHost).statusCode());@@ -357,21 +334,7 @@ class EcrRegistryDataPlaneTest {HttpServer dataPlane = null;try {- EcrRegistryManager registryManager = Mockito.mock(EcrRegistryManager.class);- EcrService ecrService = Mockito.mock(EcrService.class);- EmulatorConfig config = Mockito.mock(EmulatorConfig.class, Mockito.RETURNS_DEEP_STUBS);- when(config.services().ecr().enabled()).thenReturn(true);- when(config.services().ecr().uriStyle()).thenReturn("hostname");- when(registryManager.httpClient())- .thenReturn(new RegistryHttpClient("http://127.0.0.1:" + registry.actualPort()));- when(ecrService.registryRepositoryName("platform/api", "000000000000", "us-east-1"))- .thenReturn("legacy/platform-api");-- Router router = Router.router(vertx);- new EcrRegistryDataPlane(registryManager, ecrService, config, vertx).register(router);- dataPlane = vertx.createHttpServer().requestHandler(router)- .listen(0, "127.0.0.1")- .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ dataPlane = dataPlane(registry, false);List<String> sent = List.of("application/vnd.docker.distribution.manifest.v2+json",@@ -401,6 +364,215 @@ class EcrRegistryDataPlaneTest {}}+ /**+ * An HTTP/2 client such as crane sends every upload of a push as a stream on one+ * connection, and the data plane needs a registry connection for each. An upload left+ * waiting for one stays paused, and its unread bytes hold the client connection's+ * flow-control window: the uploads that do have a registry connection stall and never+ * give it back.+ */+ @Test+ void concurrentHttp2UploadsOnOneConnectionAllReachTheRegistry() throws Exception {+ int uploads = 8;+ Buffer layer = Buffer.buffer(new byte[1024 * 1024]);+ AtomicInteger bytesReceived = new AtomicInteger();+ CountDownLatch registryConnectionsClosed = new CountDownLatch(uploads);+ HttpServer registry = vertx.createHttpServer()+ .connectionHandler(connection -> connection.closeHandler(closed -> registryConnectionsClosed.countDown()))+ .requestHandler(request -> request.body().onSuccess(body -> {+ bytesReceived.addAndGet(body.length());+ request.response().setStatusCode(202).end();+ }))+ .listen(0, "127.0.0.1")+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);++ HttpServer dataPlane = null;+ HttpClient client = null;+ try {+ dataPlane = dataPlane(registry, false);++ // HTTP/2 with prior knowledge. The client pools a single HTTP/2 connection by+ // default, so every upload is a stream on the same connection, as crane's are.+ client = vertx.createHttpClient(new HttpClientOptions()+ .setProtocolVersion(HttpVersion.HTTP_2)+ .setHttp2ClearTextUpgrade(false));+ SocketAddress server = SocketAddress.inetSocketAddress(dataPlane.actualPort(), "127.0.0.1");+ List<Future<HttpClientResponse>> responses = new ArrayList<>();+ for (int i = 0; i < uploads; i++) {+ responses.add(client.request(new RequestOptions()+ .setServer(server)+ .setHost("000000000000.dkr.ecr.us-east-1.localhost")+ .setPort(4566)+ .setMethod(HttpMethod.PATCH)+ .setURI("/v2/platform/api/blobs/uploads/upload-" + i))+ .compose(request -> request.send(layer)));+ }++ Future.all(responses).toCompletionStage().toCompletableFuture().get(10, TimeUnit.SECONDS);+ for (Future<HttpClientResponse> response : responses) {+ assertEquals(HttpVersion.HTTP_2, response.result().version());+ assertEquals(202, response.result().statusCode());+ }+ assertEquals(uploads * layer.length(), bytesReceived.get());+ assertTrue(registryConnectionsClosed.await(5, TimeUnit.SECONDS),+ "every exchange closes the registry connection it opened");+ } finally {+ if (client != null) {+ client.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ }+ if (dataPlane != null) {+ dataPlane.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ }+ registry.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ }+ }++ /**+ * An HTTP/2 client that resets an upload's stream mid-body ends the exchange: the data plane+ * closes the registry connection it opened for it, which would otherwise wait for the rest of+ * the declared Content-Length.+ */+ @Test+ void anUploadResetMidBodyClosesItsRegistryConnection() throws Exception {+ CountDownLatch registryConnectionClosed = new CountDownLatch(1);+ CountDownLatch uploadStarted = new CountDownLatch(1);+ HttpServer registry = vertx.createHttpServer()+ .connectionHandler(connection -> connection.closeHandler(closed -> registryConnectionClosed.countDown()))+ .requestHandler(request -> request.handler(chunk -> uploadStarted.countDown()))+ .listen(0, "127.0.0.1")+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);++ HttpServer dataPlane = null;+ HttpClient client = null;+ try {+ dataPlane = dataPlane(registry, false);+ client = vertx.createHttpClient(new HttpClientOptions()+ .setProtocolVersion(HttpVersion.HTTP_2)+ .setHttp2ClearTextUpgrade(false));+ HttpClientRequest upload = client.request(new RequestOptions()+ .setServer(SocketAddress.inetSocketAddress(dataPlane.actualPort(), "127.0.0.1"))+ .setHost("000000000000.dkr.ecr.us-east-1.localhost")+ .setPort(4566)+ .setMethod(HttpMethod.PATCH)+ .setURI("/v2/platform/api/blobs/uploads/abandoned"))+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ assertEquals(HttpVersion.HTTP_2, upload.version());+ upload.putHeader("Content-Length", String.valueOf(1024 * 1024));+ upload.write(Buffer.buffer(new byte[64 * 1024]));+ assertTrue(uploadStarted.await(5, TimeUnit.SECONDS), "the upload reached the registry");++ upload.reset();++ assertTrue(registryConnectionClosed.await(5, TimeUnit.SECONDS),+ "the abandoned exchange closes its registry connection");+ } finally {+ if (client != null) {+ client.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ }+ if (dataPlane != null) {+ dataPlane.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ }+ registry.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ }+ }++ /**+ * A client that resets its stream while the data plane is still checking an immutable tag ends+ * the exchange there: nothing of it reaches the registry, and the tag is free for the next+ * write.+ */+ @Test+ void aWriteResetDuringTheTagCheckNeverReachesTheRegistryAndFreesTheTag() throws Exception {+ AtomicReference<HttpServerRequest> heldCheck = new AtomicReference<>();+ CountDownLatch checkArrived = new CountDownLatch(1);+ AtomicInteger manifestWrites = new AtomicInteger();+ AtomicReference<String> manifestBody = new AtomicReference<>();+ HttpServer registry = vertx.createHttpServer()+ .requestHandler(request -> {+ if (request.method() == HttpMethod.HEAD) {+ // The first check waits until the test answers it; later ones find no tag.+ if (heldCheck.compareAndSet(null, request)) {+ checkArrived.countDown();+ } else {+ request.response().setStatusCode(404).end();+ }+ } else if (request.method() == HttpMethod.PUT) {+ manifestWrites.incrementAndGet();+ request.body().onSuccess(body -> {+ manifestBody.set(body.toString());+ request.response().setStatusCode(201).end();+ });+ } else {+ request.response().setStatusCode(405).end();+ }+ })+ .listen(0, "127.0.0.1")+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);++ HttpServer dataPlane = null;+ HttpClient client = null;+ try {+ dataPlane = dataPlane(registry, true);+ client = vertx.createHttpClient(new HttpClientOptions()+ .setProtocolVersion(HttpVersion.HTTP_2)+ .setHttp2ClearTextUpgrade(false));+ HttpClientRequest write = client.request(new RequestOptions()+ .setServer(SocketAddress.inetSocketAddress(dataPlane.actualPort(), "127.0.0.1"))+ .setHost("000000000000.dkr.ecr.us-east-1.localhost")+ .setPort(4566)+ .setMethod(HttpMethod.PUT)+ .setURI("/v2/platform/api/manifests/v1"))+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ assertEquals(HttpVersion.HTTP_2, write.version());+ write.putHeader("Content-Length", "64");+ write.write(Buffer.buffer("{\"abandoned\""));+ assertTrue(checkArrived.await(5, TimeUnit.SECONDS), "the data plane checks the tag first");++ write.reset();+ // A PING answered after the reset means the data plane has read the reset first.+ write.connection().ping(Buffer.buffer("12345678"))+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ heldCheck.get().response().setStatusCode(404).end();++ assertEquals(201, putManifest(dataPlane.actualPort()).statusCode());+ assertEquals(1, manifestWrites.get());+ assertEquals("{}", manifestBody.get());+ } finally {+ if (client != null) {+ client.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ }+ if (dataPlane != null) {+ dataPlane.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ }+ registry.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ }+ }++ /**+ * A data plane in front of {@code registry}, on plain HTTP that a client can speak HTTP/1.1 or,+ * with prior knowledge, HTTP/2 to. Repository {@code platform/api} of account 000000000000 in+ * us-east-1 is stored as {@code legacy/platform-api}, with immutable tags when asked.+ */+ private HttpServer dataPlane(HttpServer registry, boolean immutableTags) throws Exception {+ EcrRegistryManager registryManager = Mockito.mock(EcrRegistryManager.class);+ EcrService ecrService = Mockito.mock(EcrService.class);+ EmulatorConfig config = Mockito.mock(EmulatorConfig.class, Mockito.RETURNS_DEEP_STUBS);+ when(config.services().ecr().enabled()).thenReturn(true);+ when(config.services().ecr().uriStyle()).thenReturn("hostname");+ when(registryManager.httpClient())+ .thenReturn(new RegistryHttpClient("http://127.0.0.1:" + registry.actualPort()));+ when(ecrService.isImageTagImmutable("platform/api", "000000000000", "us-east-1"))+ .thenReturn(immutableTags);+ when(ecrService.registryRepositoryName("platform/api", "000000000000", "us-east-1"))+ .thenReturn("legacy/platform-api");++ Router router = Router.router(vertx);+ new EcrRegistryDataPlane(registryManager, ecrService, config, vertx).register(router);+ return vertx.createHttpServer().requestHandler(router)+ .listen(0, "127.0.0.1")+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);+ }+private record ManifestResponse(int statusCode, String body) {}