Light, fluffy, and always free - The AWS Local Emulator alternative
fix(core): fail a Docker call after 30 seconds when its connection pool is full (#5285)
Hector Ventura985921c3db4c
diff --git a/docs/configuration/docker.md b/docs/configuration/docker.mdindex 04c3742848..438f697eed 100644--- a/docs/configuration/docker.md+++ b/docs/configuration/docker.md@@ -72,6 +72,16 @@ floci:Environment variable: `FLOCI_DOCKER_STREAMING_MAX_CONNECTIONS`+When a pool is full, a Docker call waits for a free connection, then fails. The wait defaults to 30 seconds for both pools and must be at least 1 (httpclient5 reads 0 as no limit, so a call would wait forever); a failed wait is not retried, because a full pool means Floci is saturated:++```yaml+floci:+ docker:+ connection-request-timeout-seconds: 30+```++Environment variable: `FLOCI_DOCKER_CONNECTION_REQUEST_TIMEOUT_SECONDS`+## Private Registry AuthenticationAny service that pulls a container image from a private registry (Lambda image functions, custom OpenSearch images, private Postgres images, etc.) needs Docker credentials. Two approaches are supported and can be combined.diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 77cc8f237c..8f542dd0e6 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -3516,6 +3516,15 @@ public interface EmulatorConfig {@WithDefault("512")int streamingMaxConnections();+ /**+ * How long a Docker call waits for a free pooled connection before it fails with+ * {@code ConnectionRequestTimeoutException}, in seconds, for both pools. A full pool means+ * the emulator is saturated, so failing sooner beats httpclient5's default three-minute wait;+ * the transport retry never replays a lease timeout.+ */+ @WithDefault("30")+ int connectionRequestTimeoutSeconds();+/*** Optional namespace inserted into Floci-managed child container and volume names.* Useful when multiple Floci processes share one Docker daemon.diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.javaindex 42864bca72..aecb812f73 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.java@@ -401,21 +401,29 @@ public class DockerClientProducer {* connection pool to reject it with a generic error.*/static int validateMaxConnections(int maxConnections) {- if (maxConnections < 1) {- throw new IllegalArgumentException("floci.docker.max-connections (FLOCI_DOCKER_MAX_CONNECTIONS) "- + "must be at least 1, got: " + maxConnections);- }- return maxConnections;+ return requireAtLeastOne(maxConnections, "floci.docker.max-connections", "FLOCI_DOCKER_MAX_CONNECTIONS");}/** Same check as {@link #validateMaxConnections(int)}, for the streaming pool's setting. */static int validateStreamingMaxConnections(int streamingMaxConnections) {- if (streamingMaxConnections < 1) {- throw new IllegalArgumentException("floci.docker.streaming-max-connections "- + "(FLOCI_DOCKER_STREAMING_MAX_CONNECTIONS) must be at least 1, got: "- + streamingMaxConnections);+ return requireAtLeastOne(streamingMaxConnections, "floci.docker.streaming-max-connections",+ "FLOCI_DOCKER_STREAMING_MAX_CONNECTIONS");+ }++ /**+ * Rejects a lease timeout below 1 second: httpclient5 reads zero as no timeout at all, so a+ * call would wait forever on a full pool, and a negative value fails deep inside the client.+ */+ static int validateConnectionRequestTimeoutSeconds(int seconds) {+ return requireAtLeastOne(seconds, "floci.docker.connection-request-timeout-seconds",+ "FLOCI_DOCKER_CONNECTION_REQUEST_TIMEOUT_SECONDS");+ }++ private static int requireAtLeastOne(int value, String setting, String envVar) {+ if (value < 1) {+ throw new IllegalArgumentException(setting + " (" + envVar + ") must be at least 1, got: " + value);}- return streamingMaxConnections;+ return value;}/**@@ -488,17 +496,20 @@ public class DockerClientProducer {LOG.infov("Creating {0} DockerClient pool (maxConnections={1}) for host: {2}",role, maxConnections, clientConfig.getDockerHost());- return DockerClientImpl.getInstance(clientConfig,- wrapForRole(newHttpClient(clientConfig, maxConnections), role));+ return DockerClientImpl.getInstance(clientConfig, wrapForRole(newHttpClient(clientConfig, maxConnections,+ Duration.ofSeconds(validateConnectionRequestTimeoutSeconds(+ config.docker().connectionRequestTimeoutSeconds()))), role));}- static FlociDockerHttpClient newHttpClient(DefaultDockerClientConfig clientConfig, int maxConnections) {+ static FlociDockerHttpClient newHttpClient(DefaultDockerClientConfig clientConfig, int maxConnections,+ Duration connectionRequestTimeout) {return new FlociDockerHttpClient.Builder().dockerHost(clientConfig.getDockerHost()).sslConfig(clientConfig.getSSLConfig()).maxConnections(maxConnections).connectionTimeout(Duration.ofSeconds(30)).responseTimeout(Duration.ofMinutes(5))+ .connectionRequestTimeout(connectionRequestTimeout).build();}diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/FlociDockerHttpClient.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/FlociDockerHttpClient.javaindex fb5260a9cf..adc778ba9e 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/FlociDockerHttpClient.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/FlociDockerHttpClient.java@@ -74,7 +74,8 @@ public final class FlociDockerHttpClient implements DockerHttpClient {private final String pathPrefix;private FlociDockerHttpClient(URI dockerHost, SSLConfig sslConfig, int maxConnections,- Duration connectionTimeout, Duration responseTimeout) {+ Duration connectionTimeout, Duration responseTimeout,+ Duration connectionRequestTimeout) {SSLContext sslContext;try {sslContext = sslConfig != null ? sslConfig.getSSLContext() : null;@@ -115,6 +116,8 @@ public final class FlociDockerHttpClient implements DockerHttpClient {.setDefaultRequestConfig(RequestConfig.custom().setResponseTimeout(responseTimeout != null? Timeout.of(responseTimeout.toNanos(), TimeUnit.NANOSECONDS) : null)+ .setConnectionRequestTimeout(connectionRequestTimeout != null+ ? Timeout.of(connectionRequestTimeout.toNanos(), TimeUnit.NANOSECONDS) : null).build()).disableConnectionState().build();@@ -193,6 +196,7 @@ public final class FlociDockerHttpClient implements DockerHttpClient {private int maxConnections = Integer.MAX_VALUE;private Duration connectionTimeout;private Duration responseTimeout;+ private Duration connectionRequestTimeout;public Builder dockerHost(URI value) {this.dockerHost = Objects.requireNonNull(value, "dockerHost");@@ -219,9 +223,19 @@ public final class FlociDockerHttpClient implements DockerHttpClient {return this;}+ /**+ * How long a call waits to lease a pooled connection before failing with+ * {@code ConnectionRequestTimeoutException}; unset keeps httpclient5's own default (3 minutes).+ */+ public Builder connectionRequestTimeout(Duration value) {+ this.connectionRequestTimeout = value;+ return this;+ }+public FlociDockerHttpClient build() {Objects.requireNonNull(dockerHost, "dockerHost");- return new FlociDockerHttpClient(dockerHost, sslConfig, maxConnections, connectionTimeout, responseTimeout);+ return new FlociDockerHttpClient(dockerHost, sslConfig, maxConnections, connectionTimeout, responseTimeout,+ connectionRequestTimeout);}}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex a8efe279e4..8808b4a91b 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -320,6 +320,10 @@ floci:# they never starve create/start/stop/remove. Keep in step with the @WithDefault on# EmulatorConfig.DockerConfig.streamingMaxConnections: a value set here WINS over the annotation.streaming-max-connections: 512+ # FLOCI_DOCKER_CONNECTION_REQUEST_TIMEOUT_SECONDS: how long a Docker call waits for a free pooled+ # connection before failing, for both pools. Keep in step with the @WithDefault on+ # EmulatorConfig.DockerConfig.connectionRequestTimeoutSeconds: a value set here WINS over the annotation.+ connection-request-timeout-seconds: 30resource-namespace: "" # FLOCI_DOCKER_RESOURCE_NAMESPACE; scopes child container/volume namesdocker-config-path: ""# extra-labels: # Extra Docker labels applied to every Floci-created container and volumediff --git a/src/test/java/io/github/hectorvent/floci/config/ApplicationDefaultsTest.java b/src/test/java/io/github/hectorvent/floci/config/ApplicationDefaultsTest.javaindex 8b9a10bfd5..9ebaa6154a 100644--- a/src/test/java/io/github/hectorvent/floci/config/ApplicationDefaultsTest.java+++ b/src/test/java/io/github/hectorvent/floci/config/ApplicationDefaultsTest.java@@ -152,4 +152,24 @@ class ApplicationDefaultsTest {assertEquals("512", fallback.value(),"the @WithDefault must match application.yml, which wins over it");}++ // Catches: connection-request-timeout-seconds missing from the shipped application.yml, or a yml+ // value that silently disagrees with the @WithDefault fallback.+ @Test+ void productionConfigDeclaresTheDockerLeaseTimeoutAtItsDefault() throws IOException, NoSuchMethodException {+ JsonNode leaseTimeout = new YAMLMapper()+ .readTree(Path.of("src/main/resources/application.yml").toFile())+ .path("floci").path("docker").path("connection-request-timeout-seconds");++ assertFalse(leaseTimeout.isMissingNode(),+ "connection-request-timeout-seconds should be declared in application.yml");+ assertEquals(30, leaseTimeout.asInt());++ WithDefault fallback = EmulatorConfig.DockerConfig.class+ .getMethod("connectionRequestTimeoutSeconds")+ .getAnnotation(WithDefault.class);+ assertNotNull(fallback, "connectionRequestTimeoutSeconds should declare a fallback default");+ assertEquals("30", fallback.value(),+ "the @WithDefault must match application.yml, which wins over it");+ }}diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.javaindex e2add9ad28..1685daa396 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.java@@ -12,6 +12,7 @@ import org.junit.jupiter.params.provider.MethodSource;import org.junit.jupiter.params.provider.ValueSource;import java.io.IOException;+import java.time.Duration;import java.net.InetAddress;import java.net.ServerSocket;import java.net.Socket;@@ -272,6 +273,7 @@ class DockerClientProducerTest {when(docker.dockerConfigPath()).thenReturn(Optional.empty());when(docker.maxConnections()).thenReturn(100);when(docker.streamingMaxConnections()).thenReturn(512);+ when(docker.connectionRequestTimeoutSeconds()).thenReturn(30);DockerClientProducer producer = new DockerClientProducer(config);@@ -670,6 +672,23 @@ class DockerClientProducerTest {assertEquals(1024, DockerClientProducer.validateMaxConnections(1024));}+ // Catches: a zero lease timeout reaching httpclient5, which reads it as no timeout, so a call+ // on a full pool would wait forever.+ @ParameterizedTest+ @ValueSource(ints = {0, -1})+ void validateConnectionRequestTimeoutSeconds_belowOne_throwsNamingTheSetting(int seconds) {+ IllegalArgumentException e = assertThrows(IllegalArgumentException.class,+ () -> DockerClientProducer.validateConnectionRequestTimeoutSeconds(seconds));++ assertThat(e.getMessage(), containsString("floci.docker.connection-request-timeout-seconds"));+ }++ @Test+ void validateConnectionRequestTimeoutSeconds_positive_returnsItUnchanged() {+ assertEquals(1, DockerClientProducer.validateConnectionRequestTimeoutSeconds(1));+ assertEquals(30, DockerClientProducer.validateConnectionRequestTimeoutSeconds(30));+ }+/*** The pool size comes from {@code floci.docker.max-connections}. A followed log stream holds* its connection until the container exits, and this fake daemon never answers, so with a@@ -697,6 +716,7 @@ class DockerClientProducerTest {when(docker.dockerHost()).thenReturn("tcp://127.0.0.1:" + daemon.getLocalPort());when(docker.dockerConfigPath()).thenReturn(Optional.empty());when(docker.maxConnections()).thenReturn(2);+ when(docker.connectionRequestTimeoutSeconds()).thenReturn(30);DockerClient client = new DockerClientProducer(config).dockerClient();try {@@ -740,7 +760,8 @@ class DockerClientProducerTest {// An HTTP client that actually uses the SSL config must read the certificates, so// unreadable ones fail the build; one that ignores the config builds silently.- assertThrows(RuntimeException.class, () -> DockerClientProducer.newHttpClient(tlsConfig, 10));+ assertThrows(RuntimeException.class,+ () -> DockerClientProducer.newHttpClient(tlsConfig, 10, Duration.ofSeconds(30)));}@Test@@ -752,7 +773,8 @@ class DockerClientProducerTest {.withDockerTlsVerify(false).build();- try (FlociDockerHttpClient client = DockerClientProducer.newHttpClient(plainConfig, 10)) {+ try (FlociDockerHttpClient client =+ DockerClientProducer.newHttpClient(plainConfig, 10, Duration.ofSeconds(30))) {assertNull(plainConfig.getSSLConfig(), "a config without TLS verify carries no SSL config");assertEquals("tcp://127.0.0.1:2375", plainConfig.getDockerHost().toString());}diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/FlociDockerHttpClientTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/FlociDockerHttpClientTest.javanew file mode 100644index 0000000000..0feeead6ca--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/FlociDockerHttpClientTest.java@@ -0,0 +1,76 @@+package io.github.hectorvent.floci.core.common.docker;++import com.github.dockerjava.transport.DockerHttpClient;+import org.apache.hc.core5.http.ConnectionRequestTimeoutException;+import org.junit.jupiter.api.Test;++import java.io.IOException;+import java.net.ServerSocket;+import java.net.Socket;+import java.net.URI;+import java.time.Duration;+import java.util.List;+import java.util.concurrent.CopyOnWriteArrayList;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++class FlociDockerHttpClientTest {++ // Catches: a full pool making a call wait out httpclient5's three-minute default instead of+ // failing after the configured lease timeout.+ @Test+ void aCallThatCannotLeaseAConnectionFailsAfterTheLeaseTimeout() throws Exception {+ List<Socket> accepted = new CopyOnWriteArrayList<>();+ CountDownLatch firstConnected = new CountDownLatch(1);+ try (ServerSocket server = new ServerSocket(0)) {+ Thread acceptor = Thread.ofVirtual().start(() -> {+ try {+ while (true) {+ accepted.add(server.accept());+ firstConnected.countDown();+ }+ } catch (IOException expected) {+ // The server socket closed at the end of the test.+ }+ });+ try (FlociDockerHttpClient client = new FlociDockerHttpClient.Builder()+ .dockerHost(URI.create("tcp://127.0.0.1:" + server.getLocalPort()))+ .maxConnections(1)+ .responseTimeout(Duration.ofSeconds(30))+ .connectionRequestTimeout(Duration.ofSeconds(1))+ .build()) {+ // The one pooled connection is held by a request the server never answers.+ Thread.ofVirtual().start(() -> {+ try (DockerHttpClient.Response ignored = client.execute(get("/containers/held/wait"))) {+ // Never reached: the server does not answer before the test ends.+ } catch (RuntimeException expected) {+ // The client closed when the test ended.+ }+ });+ assertTrue(firstConnected.await(5, TimeUnit.SECONDS), "the first request should connect");++ long start = System.nanoTime();+ RuntimeException failure = assertThrows(RuntimeException.class,+ () -> client.execute(get("/_ping")));+ long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);++ assertTrue(failure.getCause() instanceof ConnectionRequestTimeoutException,+ "a full pool should fail with ConnectionRequestTimeoutException, got " + failure.getCause());+ assertTrue(elapsedMillis < 10_000,+ "should fail near the 1 s lease timeout, took " + elapsedMillis + " ms");+ } finally {+ for (Socket socket : accepted) {+ socket.close();+ }+ acceptor.interrupt();+ }+ }+ }++ private static DockerHttpClient.Request get(String path) {+ return DockerHttpClient.Request.builder().method(DockerHttpClient.Request.Method.GET).path(path).build();+ }+}