Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ecr): give every data plane exchange its own registry connection (#5883)

Alessandro Lattaode94e3cfcc03
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlane.java b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlane.java
index d67b811bdf..2b2b7fdc6a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlane.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlane.java
@@ -24,6 +24,7 @@ import java.nio.charset.StandardCharsets;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -45,7 +46,6 @@ public class EcrRegistryDataPlane {
private final EcrService ecrService;
private final EmulatorConfig config;
private final Vertx vertx;
- private final HttpClient proxyClient;
private final Set<String> inFlightImmutableTags = ConcurrentHashMap.newKeySet();
@Inject
@@ -57,9 +57,6 @@ public class EcrRegistryDataPlane {
this.ecrService = ecrService;
this.config = config;
this.vertx = vertx;
- this.proxyClient = vertx.createHttpClient(new HttpClientOptions()
- .setConnectTimeout(5_000)
- .setKeepAlive(true));
}
void register(@Observes Router router) {
@@ -91,7 +88,14 @@ public class EcrRegistryDataPlane {
return;
}
- checkTagAndProxy(context, registryRequest, immutableTag);
+ Exchange exchange = new Exchange(immutableTag);
+ // A client that goes away, by resetting its HTTP/2 stream or closing the connection, ends
+ // the exchange at any point, the registry check included: otherwise the registry keeps
+ // waiting for the rest of a body with a Content-Length, and the exchange's connection,
+ // client and immutable tag lock stay.
+ context.response().closeHandler(closed -> exchange.end());
+ context.request().exceptionHandler(failure -> exchange.end());
+ checkTagAndProxy(context, registryRequest, exchange);
}
private String immutableTag(RoutingContext context, RegistryRequest request) {
@@ -102,7 +106,8 @@ public class EcrRegistryDataPlane {
return request.accountId() + "/" + request.region() + "/" + request.repositoryName() + ":" + request.tag();
}
- private void checkTagAndProxy(RoutingContext context, RegistryRequest request, String immutableTag) {
+ private void checkTagAndProxy(RoutingContext context, RegistryRequest request, Exchange exchange) {
+ String immutableTag = exchange.immutableTag;
vertx.<Boolean>executeBlocking(promise -> {
try {
registryManager.ensureStarted();
@@ -118,57 +123,72 @@ public class EcrRegistryDataPlane {
promise.fail(e);
}
}).onComplete(result -> {
+ if (exchange.ended()) {
+ // The client went away during the check: nothing of this exchange reaches the
+ // registry.
+ return;
+ }
if (result.failed()) {
context.request().resume();
- release(immutableTag);
+ exchange.end();
context.response().setStatusCode(503).end();
} else if (result.result()) {
context.request().resume();
- release(immutableTag);
+ exchange.end();
imageTagAlreadyExists(context, request);
} else {
- proxy(context, request, immutableTag);
+ proxy(context, request, exchange);
}
});
}
- private void proxy(RoutingContext context, RegistryRequest request, String immutableTag) {
+ private void proxy(RoutingContext context, RegistryRequest request, Exchange exchange) {
URI backend = URI.create(registryManager.httpClient().baseUrl());
RequestOptions options = new RequestOptions()
.setHost(backend.getHost())
.setPort(backend.getPort() == -1 ? 80 : backend.getPort())
.setURI(request.backendUri())
.setMethod(context.request().method());
- proxyClient.request(options).onComplete(upstreamRequest -> {
+ // A registry connection of its own for every exchange, never a place in a shared pool's
+ // queue. An HTTP/2 client such as crane sends a whole push as streams on one connection,
+ // so a pool of any size can be full of its uploads while another of its streams waits,
+ // paused. The waiting stream's unread bytes hold that connection's flow-control window,
+ // the uploads that do have a registry connection stop receiving data, and none of them
+ // ever gives its connection back.
+ HttpClient client = exchange.open(vertx.createHttpClient(new HttpClientOptions().setConnectTimeout(5_000)));
+ client.request(options).onComplete(upstreamRequest -> {
if (upstreamRequest.failed()) {
context.request().resume();
- release(immutableTag);
- context.response().setStatusCode(503).end();
+ exchange.end();
+ failUnlessGone(context, 503);
return;
}
HttpClientRequest clientReq = upstreamRequest.result();
copyRequestHeaders(context, clientReq);
clientReq.response().onComplete(upstreamResponse -> {
if (upstreamResponse.failed()) {
- release(immutableTag);
- if (!context.response().ended()) {
- context.response().setStatusCode(502).end();
- }
+ exchange.end();
+ failUnlessGone(context, 502);
return;
}
copyResponseHeaders(context, upstreamResponse.result(), request);
- upstreamResponse.result().pipeTo(context.response()).onComplete(ignored -> release(immutableTag));
+ upstreamResponse.result().pipeTo(context.response()).onComplete(ignored -> exchange.end());
});
clientReq.send(context.request()).onFailure(ignored -> {
- release(immutableTag);
- if (!context.response().ended()) {
- context.response().setStatusCode(502).end();
- }
+ exchange.end();
+ failUnlessGone(context, 502);
});
context.request().resume();
});
}
+ /** Answers {@code status} unless the response is already written or its client went away. */
+ private static void failUnlessGone(RoutingContext context, int status) {
+ if (!context.response().ended() && !context.response().closed()) {
+ context.response().setStatusCode(status).end();
+ }
+ }
+
private static void copyRequestHeaders(RoutingContext context, HttpClientRequest upstream) {
context.request().headers().forEach(header -> {
if (!HOP_BY_HOP_HEADERS.contains(header.getKey().toLowerCase())
@@ -226,6 +246,46 @@ public class EcrRegistryDataPlane {
}
}
+ /**
+ * One proxied request: the registry client it opens once it gets that far, and the immutable
+ * tag it holds. It ends once, at whichever comes first of the registry's answer, a failure and
+ * the client going away; the close handler also fires after an exchange that ended normally,
+ * when another exchange may already hold the same tag.
+ */
+ private final class Exchange {
+
+ private final String immutableTag;
+ private final AtomicBoolean ended = new AtomicBoolean();
+ private HttpClient client;
+
+ private Exchange(String immutableTag) {
+ this.immutableTag = immutableTag;
+ }
+
+ private boolean ended() {
+ return ended.get();
+ }
+
+ private HttpClient open(HttpClient registryClient) {
+ client = registryClient;
+ return registryClient;
+ }
+
+ private void end() {
+ if (!ended.compareAndSet(false, true)) {
+ return;
+ }
+ if (client == null) {
+ release(immutableTag);
+ return;
+ }
+ // Closing the client closes the exchange's registry connection, which the registry
+ // sees and aborts on. The tag is free only once it is closed, so another write of the
+ // tag cannot start its check while this one can still reach the registry.
+ client.close().onComplete(closed -> release(immutableTag));
+ }
+ }
+
static Optional<RegistryRequest> requestFor(String host, String path, String query, String uriStyle) {
String registryPath = "/v2".equals(path) ? "/v2/" : path;
if (!registryPath.startsWith("/v2/")) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlaneTest.java b/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlaneTest.java
index 507366d761..81c622e05f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlaneTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryDataPlaneTest.java
@@ -2,21 +2,30 @@ package io.github.hectorvent.floci.services.ecr.registry;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.services.ecr.EcrService;
+import io.vertx.core.Future;
import io.vertx.core.Vertx;
import io.vertx.core.buffer.Buffer;
import io.vertx.core.http.HttpClient;
+import io.vertx.core.http.HttpClientOptions;
+import io.vertx.core.http.HttpClientRequest;
+import io.vertx.core.http.HttpClientResponse;
import io.vertx.core.http.HttpMethod;
import io.vertx.core.http.HttpServer;
+import io.vertx.core.http.HttpServerRequest;
+import io.vertx.core.http.HttpVersion;
import io.vertx.core.http.RequestOptions;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
+import io.vertx.core.net.SocketAddress;
import io.vertx.ext.web.Router;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
+import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -243,23 +252,7 @@ class EcrRegistryDataPlaneTest {
HttpServer dataPlane = null;
try {
- EcrRegistryManager registryManager = Mockito.mock(EcrRegistryManager.class);
- EcrService ecrService = Mockito.mock(EcrService.class);
- EmulatorConfig config = Mockito.mock(EmulatorConfig.class, Mockito.RETURNS_DEEP_STUBS);
- when(config.services().ecr().enabled()).thenReturn(true);
- when(config.services().ecr().uriStyle()).thenReturn("hostname");
- when(registryManager.httpClient())
- .thenReturn(new RegistryHttpClient("http://127.0.0.1:" + registry.actualPort()));
- when(ecrService.isImageTagImmutable("platform/api", "000000000000", "us-east-1"))
- .thenReturn(true);
- when(ecrService.registryRepositoryName("platform/api", "000000000000", "us-east-1"))
- .thenReturn("legacy/platform-api");
-
- Router router = Router.router(vertx);
- new EcrRegistryDataPlane(registryManager, ecrService, config, vertx).register(router);
- dataPlane = vertx.createHttpServer().requestHandler(router)
- .listen(0, "127.0.0.1")
- .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ dataPlane = dataPlane(registry, true);
assertEquals(201, putManifest(dataPlane.actualPort()).statusCode());
@@ -307,23 +300,7 @@ class EcrRegistryDataPlaneTest {
HttpServer dataPlane = null;
try {
- EcrRegistryManager registryManager = Mockito.mock(EcrRegistryManager.class);
- EcrService ecrService = Mockito.mock(EcrService.class);
- EmulatorConfig config = Mockito.mock(EmulatorConfig.class, Mockito.RETURNS_DEEP_STUBS);
- when(config.services().ecr().enabled()).thenReturn(true);
- when(config.services().ecr().uriStyle()).thenReturn("hostname");
- when(registryManager.httpClient())
- .thenReturn(new RegistryHttpClient("http://127.0.0.1:" + registry.actualPort()));
- when(ecrService.isImageTagImmutable("platform/api", "000000000000", "us-east-1"))
- .thenReturn(true);
- when(ecrService.registryRepositoryName("platform/api", "000000000000", "us-east-1"))
- .thenReturn("legacy/platform-api");
-
- Router router = Router.router(vertx);
- new EcrRegistryDataPlane(registryManager, ecrService, config, vertx).register(router);
- dataPlane = vertx.createHttpServer().requestHandler(router)
- .listen(0, "127.0.0.1")
- .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ dataPlane = dataPlane(registry, true);
String tlsHost = "000000000000.dkr.ecr.us-east-1.localhost.floci.io:4566";
assertEquals(201, putManifest(dataPlane.actualPort(), tlsHost).statusCode());
@@ -357,21 +334,7 @@ class EcrRegistryDataPlaneTest {
HttpServer dataPlane = null;
try {
- EcrRegistryManager registryManager = Mockito.mock(EcrRegistryManager.class);
- EcrService ecrService = Mockito.mock(EcrService.class);
- EmulatorConfig config = Mockito.mock(EmulatorConfig.class, Mockito.RETURNS_DEEP_STUBS);
- when(config.services().ecr().enabled()).thenReturn(true);
- when(config.services().ecr().uriStyle()).thenReturn("hostname");
- when(registryManager.httpClient())
- .thenReturn(new RegistryHttpClient("http://127.0.0.1:" + registry.actualPort()));
- when(ecrService.registryRepositoryName("platform/api", "000000000000", "us-east-1"))
- .thenReturn("legacy/platform-api");
-
- Router router = Router.router(vertx);
- new EcrRegistryDataPlane(registryManager, ecrService, config, vertx).register(router);
- dataPlane = vertx.createHttpServer().requestHandler(router)
- .listen(0, "127.0.0.1")
- .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ dataPlane = dataPlane(registry, false);
List<String> sent = List.of(
"application/vnd.docker.distribution.manifest.v2+json",
@@ -401,6 +364,215 @@ class EcrRegistryDataPlaneTest {
}
}
+ /**
+ * An HTTP/2 client such as crane sends every upload of a push as a stream on one
+ * connection, and the data plane needs a registry connection for each. An upload left
+ * waiting for one stays paused, and its unread bytes hold the client connection's
+ * flow-control window: the uploads that do have a registry connection stall and never
+ * give it back.
+ */
+ @Test
+ void concurrentHttp2UploadsOnOneConnectionAllReachTheRegistry() throws Exception {
+ int uploads = 8;
+ Buffer layer = Buffer.buffer(new byte[1024 * 1024]);
+ AtomicInteger bytesReceived = new AtomicInteger();
+ CountDownLatch registryConnectionsClosed = new CountDownLatch(uploads);
+ HttpServer registry = vertx.createHttpServer()
+ .connectionHandler(connection -> connection.closeHandler(closed -> registryConnectionsClosed.countDown()))
+ .requestHandler(request -> request.body().onSuccess(body -> {
+ bytesReceived.addAndGet(body.length());
+ request.response().setStatusCode(202).end();
+ }))
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ HttpServer dataPlane = null;
+ HttpClient client = null;
+ try {
+ dataPlane = dataPlane(registry, false);
+
+ // HTTP/2 with prior knowledge. The client pools a single HTTP/2 connection by
+ // default, so every upload is a stream on the same connection, as crane's are.
+ client = vertx.createHttpClient(new HttpClientOptions()
+ .setProtocolVersion(HttpVersion.HTTP_2)
+ .setHttp2ClearTextUpgrade(false));
+ SocketAddress server = SocketAddress.inetSocketAddress(dataPlane.actualPort(), "127.0.0.1");
+ List<Future<HttpClientResponse>> responses = new ArrayList<>();
+ for (int i = 0; i < uploads; i++) {
+ responses.add(client.request(new RequestOptions()
+ .setServer(server)
+ .setHost("000000000000.dkr.ecr.us-east-1.localhost")
+ .setPort(4566)
+ .setMethod(HttpMethod.PATCH)
+ .setURI("/v2/platform/api/blobs/uploads/upload-" + i))
+ .compose(request -> request.send(layer)));
+ }
+
+ Future.all(responses).toCompletionStage().toCompletableFuture().get(10, TimeUnit.SECONDS);
+ for (Future<HttpClientResponse> response : responses) {
+ assertEquals(HttpVersion.HTTP_2, response.result().version());
+ assertEquals(202, response.result().statusCode());
+ }
+ assertEquals(uploads * layer.length(), bytesReceived.get());
+ assertTrue(registryConnectionsClosed.await(5, TimeUnit.SECONDS),
+ "every exchange closes the registry connection it opened");
+ } finally {
+ if (client != null) {
+ client.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ }
+ if (dataPlane != null) {
+ dataPlane.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ }
+ registry.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ }
+ }
+
+ /**
+ * An HTTP/2 client that resets an upload's stream mid-body ends the exchange: the data plane
+ * closes the registry connection it opened for it, which would otherwise wait for the rest of
+ * the declared Content-Length.
+ */
+ @Test
+ void anUploadResetMidBodyClosesItsRegistryConnection() throws Exception {
+ CountDownLatch registryConnectionClosed = new CountDownLatch(1);
+ CountDownLatch uploadStarted = new CountDownLatch(1);
+ HttpServer registry = vertx.createHttpServer()
+ .connectionHandler(connection -> connection.closeHandler(closed -> registryConnectionClosed.countDown()))
+ .requestHandler(request -> request.handler(chunk -> uploadStarted.countDown()))
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ HttpServer dataPlane = null;
+ HttpClient client = null;
+ try {
+ dataPlane = dataPlane(registry, false);
+ client = vertx.createHttpClient(new HttpClientOptions()
+ .setProtocolVersion(HttpVersion.HTTP_2)
+ .setHttp2ClearTextUpgrade(false));
+ HttpClientRequest upload = client.request(new RequestOptions()
+ .setServer(SocketAddress.inetSocketAddress(dataPlane.actualPort(), "127.0.0.1"))
+ .setHost("000000000000.dkr.ecr.us-east-1.localhost")
+ .setPort(4566)
+ .setMethod(HttpMethod.PATCH)
+ .setURI("/v2/platform/api/blobs/uploads/abandoned"))
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ assertEquals(HttpVersion.HTTP_2, upload.version());
+ upload.putHeader("Content-Length", String.valueOf(1024 * 1024));
+ upload.write(Buffer.buffer(new byte[64 * 1024]));
+ assertTrue(uploadStarted.await(5, TimeUnit.SECONDS), "the upload reached the registry");
+
+ upload.reset();
+
+ assertTrue(registryConnectionClosed.await(5, TimeUnit.SECONDS),
+ "the abandoned exchange closes its registry connection");
+ } finally {
+ if (client != null) {
+ client.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ }
+ if (dataPlane != null) {
+ dataPlane.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ }
+ registry.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ }
+ }
+
+ /**
+ * A client that resets its stream while the data plane is still checking an immutable tag ends
+ * the exchange there: nothing of it reaches the registry, and the tag is free for the next
+ * write.
+ */
+ @Test
+ void aWriteResetDuringTheTagCheckNeverReachesTheRegistryAndFreesTheTag() throws Exception {
+ AtomicReference<HttpServerRequest> heldCheck = new AtomicReference<>();
+ CountDownLatch checkArrived = new CountDownLatch(1);
+ AtomicInteger manifestWrites = new AtomicInteger();
+ AtomicReference<String> manifestBody = new AtomicReference<>();
+ HttpServer registry = vertx.createHttpServer()
+ .requestHandler(request -> {
+ if (request.method() == HttpMethod.HEAD) {
+ // The first check waits until the test answers it; later ones find no tag.
+ if (heldCheck.compareAndSet(null, request)) {
+ checkArrived.countDown();
+ } else {
+ request.response().setStatusCode(404).end();
+ }
+ } else if (request.method() == HttpMethod.PUT) {
+ manifestWrites.incrementAndGet();
+ request.body().onSuccess(body -> {
+ manifestBody.set(body.toString());
+ request.response().setStatusCode(201).end();
+ });
+ } else {
+ request.response().setStatusCode(405).end();
+ }
+ })
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ HttpServer dataPlane = null;
+ HttpClient client = null;
+ try {
+ dataPlane = dataPlane(registry, true);
+ client = vertx.createHttpClient(new HttpClientOptions()
+ .setProtocolVersion(HttpVersion.HTTP_2)
+ .setHttp2ClearTextUpgrade(false));
+ HttpClientRequest write = client.request(new RequestOptions()
+ .setServer(SocketAddress.inetSocketAddress(dataPlane.actualPort(), "127.0.0.1"))
+ .setHost("000000000000.dkr.ecr.us-east-1.localhost")
+ .setPort(4566)
+ .setMethod(HttpMethod.PUT)
+ .setURI("/v2/platform/api/manifests/v1"))
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ assertEquals(HttpVersion.HTTP_2, write.version());
+ write.putHeader("Content-Length", "64");
+ write.write(Buffer.buffer("{\"abandoned\""));
+ assertTrue(checkArrived.await(5, TimeUnit.SECONDS), "the data plane checks the tag first");
+
+ write.reset();
+ // A PING answered after the reset means the data plane has read the reset first.
+ write.connection().ping(Buffer.buffer("12345678"))
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ heldCheck.get().response().setStatusCode(404).end();
+
+ assertEquals(201, putManifest(dataPlane.actualPort()).statusCode());
+ assertEquals(1, manifestWrites.get());
+ assertEquals("{}", manifestBody.get());
+ } finally {
+ if (client != null) {
+ client.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ }
+ if (dataPlane != null) {
+ dataPlane.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ }
+ registry.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ }
+ }
+
+ /**
+ * A data plane in front of {@code registry}, on plain HTTP that a client can speak HTTP/1.1 or,
+ * with prior knowledge, HTTP/2 to. Repository {@code platform/api} of account 000000000000 in
+ * us-east-1 is stored as {@code legacy/platform-api}, with immutable tags when asked.
+ */
+ private HttpServer dataPlane(HttpServer registry, boolean immutableTags) throws Exception {
+ EcrRegistryManager registryManager = Mockito.mock(EcrRegistryManager.class);
+ EcrService ecrService = Mockito.mock(EcrService.class);
+ EmulatorConfig config = Mockito.mock(EmulatorConfig.class, Mockito.RETURNS_DEEP_STUBS);
+ when(config.services().ecr().enabled()).thenReturn(true);
+ when(config.services().ecr().uriStyle()).thenReturn("hostname");
+ when(registryManager.httpClient())
+ .thenReturn(new RegistryHttpClient("http://127.0.0.1:" + registry.actualPort()));
+ when(ecrService.isImageTagImmutable("platform/api", "000000000000", "us-east-1"))
+ .thenReturn(immutableTags);
+ when(ecrService.registryRepositoryName("platform/api", "000000000000", "us-east-1"))
+ .thenReturn("legacy/platform-api");
+
+ Router router = Router.router(vertx);
+ new EcrRegistryDataPlane(registryManager, ecrService, config, vertx).register(router);
+ return vertx.createHttpServer().requestHandler(router)
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ }
+
private record ManifestResponse(int statusCode, String body) {
}