Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(core): fail a Docker call after 30 seconds when its connection pool is full (#5285)

Hector Ventura985921c3db4c
diff --git a/docs/configuration/docker.md b/docs/configuration/docker.md
index 04c3742848..438f697eed 100644
--- a/docs/configuration/docker.md
+++ b/docs/configuration/docker.md
@@ -72,6 +72,16 @@ floci:
Environment variable: `FLOCI_DOCKER_STREAMING_MAX_CONNECTIONS`
+When a pool is full, a Docker call waits for a free connection, then fails. The wait defaults to 30 seconds for both pools and must be at least 1 (httpclient5 reads 0 as no limit, so a call would wait forever); a failed wait is not retried, because a full pool means Floci is saturated:
+
+```yaml
+floci:
+ docker:
+ connection-request-timeout-seconds: 30
+```
+
+Environment variable: `FLOCI_DOCKER_CONNECTION_REQUEST_TIMEOUT_SECONDS`
+
## Private Registry Authentication
Any service that pulls a container image from a private registry (Lambda image functions, custom OpenSearch images, private Postgres images, etc.) needs Docker credentials. Two approaches are supported and can be combined.
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 77cc8f237c..8f542dd0e6 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -3516,6 +3516,15 @@ public interface EmulatorConfig {
@WithDefault("512")
int streamingMaxConnections();
+ /**
+ * How long a Docker call waits for a free pooled connection before it fails with
+ * {@code ConnectionRequestTimeoutException}, in seconds, for both pools. A full pool means
+ * the emulator is saturated, so failing sooner beats httpclient5's default three-minute wait;
+ * the transport retry never replays a lease timeout.
+ */
+ @WithDefault("30")
+ int connectionRequestTimeoutSeconds();
+
/**
* Optional namespace inserted into Floci-managed child container and volume names.
* Useful when multiple Floci processes share one Docker daemon.
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.java
index 42864bca72..aecb812f73 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.java
@@ -401,21 +401,29 @@ public class DockerClientProducer {
* connection pool to reject it with a generic error.
*/
static int validateMaxConnections(int maxConnections) {
- if (maxConnections < 1) {
- throw new IllegalArgumentException("floci.docker.max-connections (FLOCI_DOCKER_MAX_CONNECTIONS) "
- + "must be at least 1, got: " + maxConnections);
- }
- return maxConnections;
+ return requireAtLeastOne(maxConnections, "floci.docker.max-connections", "FLOCI_DOCKER_MAX_CONNECTIONS");
}
/** Same check as {@link #validateMaxConnections(int)}, for the streaming pool's setting. */
static int validateStreamingMaxConnections(int streamingMaxConnections) {
- if (streamingMaxConnections < 1) {
- throw new IllegalArgumentException("floci.docker.streaming-max-connections "
- + "(FLOCI_DOCKER_STREAMING_MAX_CONNECTIONS) must be at least 1, got: "
- + streamingMaxConnections);
+ return requireAtLeastOne(streamingMaxConnections, "floci.docker.streaming-max-connections",
+ "FLOCI_DOCKER_STREAMING_MAX_CONNECTIONS");
+ }
+
+ /**
+ * Rejects a lease timeout below 1 second: httpclient5 reads zero as no timeout at all, so a
+ * call would wait forever on a full pool, and a negative value fails deep inside the client.
+ */
+ static int validateConnectionRequestTimeoutSeconds(int seconds) {
+ return requireAtLeastOne(seconds, "floci.docker.connection-request-timeout-seconds",
+ "FLOCI_DOCKER_CONNECTION_REQUEST_TIMEOUT_SECONDS");
+ }
+
+ private static int requireAtLeastOne(int value, String setting, String envVar) {
+ if (value < 1) {
+ throw new IllegalArgumentException(setting + " (" + envVar + ") must be at least 1, got: " + value);
}
- return streamingMaxConnections;
+ return value;
}
/**
@@ -488,17 +496,20 @@ public class DockerClientProducer {
LOG.infov("Creating {0} DockerClient pool (maxConnections={1}) for host: {2}",
role, maxConnections, clientConfig.getDockerHost());
- return DockerClientImpl.getInstance(clientConfig,
- wrapForRole(newHttpClient(clientConfig, maxConnections), role));
+ return DockerClientImpl.getInstance(clientConfig, wrapForRole(newHttpClient(clientConfig, maxConnections,
+ Duration.ofSeconds(validateConnectionRequestTimeoutSeconds(
+ config.docker().connectionRequestTimeoutSeconds()))), role));
}
- static FlociDockerHttpClient newHttpClient(DefaultDockerClientConfig clientConfig, int maxConnections) {
+ static FlociDockerHttpClient newHttpClient(DefaultDockerClientConfig clientConfig, int maxConnections,
+ Duration connectionRequestTimeout) {
return new FlociDockerHttpClient.Builder()
.dockerHost(clientConfig.getDockerHost())
.sslConfig(clientConfig.getSSLConfig())
.maxConnections(maxConnections)
.connectionTimeout(Duration.ofSeconds(30))
.responseTimeout(Duration.ofMinutes(5))
+ .connectionRequestTimeout(connectionRequestTimeout)
.build();
}
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/FlociDockerHttpClient.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/FlociDockerHttpClient.java
index fb5260a9cf..adc778ba9e 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/FlociDockerHttpClient.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/FlociDockerHttpClient.java
@@ -74,7 +74,8 @@ public final class FlociDockerHttpClient implements DockerHttpClient {
private final String pathPrefix;
private FlociDockerHttpClient(URI dockerHost, SSLConfig sslConfig, int maxConnections,
- Duration connectionTimeout, Duration responseTimeout) {
+ Duration connectionTimeout, Duration responseTimeout,
+ Duration connectionRequestTimeout) {
SSLContext sslContext;
try {
sslContext = sslConfig != null ? sslConfig.getSSLContext() : null;
@@ -115,6 +116,8 @@ public final class FlociDockerHttpClient implements DockerHttpClient {
.setDefaultRequestConfig(RequestConfig.custom()
.setResponseTimeout(responseTimeout != null
? Timeout.of(responseTimeout.toNanos(), TimeUnit.NANOSECONDS) : null)
+ .setConnectionRequestTimeout(connectionRequestTimeout != null
+ ? Timeout.of(connectionRequestTimeout.toNanos(), TimeUnit.NANOSECONDS) : null)
.build())
.disableConnectionState()
.build();
@@ -193,6 +196,7 @@ public final class FlociDockerHttpClient implements DockerHttpClient {
private int maxConnections = Integer.MAX_VALUE;
private Duration connectionTimeout;
private Duration responseTimeout;
+ private Duration connectionRequestTimeout;
public Builder dockerHost(URI value) {
this.dockerHost = Objects.requireNonNull(value, "dockerHost");
@@ -219,9 +223,19 @@ public final class FlociDockerHttpClient implements DockerHttpClient {
return this;
}
+ /**
+ * How long a call waits to lease a pooled connection before failing with
+ * {@code ConnectionRequestTimeoutException}; unset keeps httpclient5's own default (3 minutes).
+ */
+ public Builder connectionRequestTimeout(Duration value) {
+ this.connectionRequestTimeout = value;
+ return this;
+ }
+
public FlociDockerHttpClient build() {
Objects.requireNonNull(dockerHost, "dockerHost");
- return new FlociDockerHttpClient(dockerHost, sslConfig, maxConnections, connectionTimeout, responseTimeout);
+ return new FlociDockerHttpClient(dockerHost, sslConfig, maxConnections, connectionTimeout, responseTimeout,
+ connectionRequestTimeout);
}
}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index a8efe279e4..8808b4a91b 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -320,6 +320,10 @@ floci:
# they never starve create/start/stop/remove. Keep in step with the @WithDefault on
# EmulatorConfig.DockerConfig.streamingMaxConnections: a value set here WINS over the annotation.
streaming-max-connections: 512
+ # FLOCI_DOCKER_CONNECTION_REQUEST_TIMEOUT_SECONDS: how long a Docker call waits for a free pooled
+ # connection before failing, for both pools. Keep in step with the @WithDefault on
+ # EmulatorConfig.DockerConfig.connectionRequestTimeoutSeconds: a value set here WINS over the annotation.
+ connection-request-timeout-seconds: 30
resource-namespace: "" # FLOCI_DOCKER_RESOURCE_NAMESPACE; scopes child container/volume names
docker-config-path: ""
# extra-labels: # Extra Docker labels applied to every Floci-created container and volume
diff --git a/src/test/java/io/github/hectorvent/floci/config/ApplicationDefaultsTest.java b/src/test/java/io/github/hectorvent/floci/config/ApplicationDefaultsTest.java
index 8b9a10bfd5..9ebaa6154a 100644
--- a/src/test/java/io/github/hectorvent/floci/config/ApplicationDefaultsTest.java
+++ b/src/test/java/io/github/hectorvent/floci/config/ApplicationDefaultsTest.java
@@ -152,4 +152,24 @@ class ApplicationDefaultsTest {
assertEquals("512", fallback.value(),
"the @WithDefault must match application.yml, which wins over it");
}
+
+ // Catches: connection-request-timeout-seconds missing from the shipped application.yml, or a yml
+ // value that silently disagrees with the @WithDefault fallback.
+ @Test
+ void productionConfigDeclaresTheDockerLeaseTimeoutAtItsDefault() throws IOException, NoSuchMethodException {
+ JsonNode leaseTimeout = new YAMLMapper()
+ .readTree(Path.of("src/main/resources/application.yml").toFile())
+ .path("floci").path("docker").path("connection-request-timeout-seconds");
+
+ assertFalse(leaseTimeout.isMissingNode(),
+ "connection-request-timeout-seconds should be declared in application.yml");
+ assertEquals(30, leaseTimeout.asInt());
+
+ WithDefault fallback = EmulatorConfig.DockerConfig.class
+ .getMethod("connectionRequestTimeoutSeconds")
+ .getAnnotation(WithDefault.class);
+ assertNotNull(fallback, "connectionRequestTimeoutSeconds should declare a fallback default");
+ assertEquals("30", fallback.value(),
+ "the @WithDefault must match application.yml, which wins over it");
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.java
index e2add9ad28..1685daa396 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.java
@@ -12,6 +12,7 @@ import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
+import java.time.Duration;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
@@ -272,6 +273,7 @@ class DockerClientProducerTest {
when(docker.dockerConfigPath()).thenReturn(Optional.empty());
when(docker.maxConnections()).thenReturn(100);
when(docker.streamingMaxConnections()).thenReturn(512);
+ when(docker.connectionRequestTimeoutSeconds()).thenReturn(30);
DockerClientProducer producer = new DockerClientProducer(config);
@@ -670,6 +672,23 @@ class DockerClientProducerTest {
assertEquals(1024, DockerClientProducer.validateMaxConnections(1024));
}
+ // Catches: a zero lease timeout reaching httpclient5, which reads it as no timeout, so a call
+ // on a full pool would wait forever.
+ @ParameterizedTest
+ @ValueSource(ints = {0, -1})
+ void validateConnectionRequestTimeoutSeconds_belowOne_throwsNamingTheSetting(int seconds) {
+ IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> DockerClientProducer.validateConnectionRequestTimeoutSeconds(seconds));
+
+ assertThat(e.getMessage(), containsString("floci.docker.connection-request-timeout-seconds"));
+ }
+
+ @Test
+ void validateConnectionRequestTimeoutSeconds_positive_returnsItUnchanged() {
+ assertEquals(1, DockerClientProducer.validateConnectionRequestTimeoutSeconds(1));
+ assertEquals(30, DockerClientProducer.validateConnectionRequestTimeoutSeconds(30));
+ }
+
/**
* The pool size comes from {@code floci.docker.max-connections}. A followed log stream holds
* its connection until the container exits, and this fake daemon never answers, so with a
@@ -697,6 +716,7 @@ class DockerClientProducerTest {
when(docker.dockerHost()).thenReturn("tcp://127.0.0.1:" + daemon.getLocalPort());
when(docker.dockerConfigPath()).thenReturn(Optional.empty());
when(docker.maxConnections()).thenReturn(2);
+ when(docker.connectionRequestTimeoutSeconds()).thenReturn(30);
DockerClient client = new DockerClientProducer(config).dockerClient();
try {
@@ -740,7 +760,8 @@ class DockerClientProducerTest {
// An HTTP client that actually uses the SSL config must read the certificates, so
// unreadable ones fail the build; one that ignores the config builds silently.
- assertThrows(RuntimeException.class, () -> DockerClientProducer.newHttpClient(tlsConfig, 10));
+ assertThrows(RuntimeException.class,
+ () -> DockerClientProducer.newHttpClient(tlsConfig, 10, Duration.ofSeconds(30)));
}
@Test
@@ -752,7 +773,8 @@ class DockerClientProducerTest {
.withDockerTlsVerify(false)
.build();
- try (FlociDockerHttpClient client = DockerClientProducer.newHttpClient(plainConfig, 10)) {
+ try (FlociDockerHttpClient client =
+ DockerClientProducer.newHttpClient(plainConfig, 10, Duration.ofSeconds(30))) {
assertNull(plainConfig.getSSLConfig(), "a config without TLS verify carries no SSL config");
assertEquals("tcp://127.0.0.1:2375", plainConfig.getDockerHost().toString());
}
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/FlociDockerHttpClientTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/FlociDockerHttpClientTest.java
new file mode 100644
index 0000000000..0feeead6ca
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/FlociDockerHttpClientTest.java
@@ -0,0 +1,76 @@
+package io.github.hectorvent.floci.core.common.docker;
+
+import com.github.dockerjava.transport.DockerHttpClient;
+import org.apache.hc.core5.http.ConnectionRequestTimeoutException;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URI;
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class FlociDockerHttpClientTest {
+
+ // Catches: a full pool making a call wait out httpclient5's three-minute default instead of
+ // failing after the configured lease timeout.
+ @Test
+ void aCallThatCannotLeaseAConnectionFailsAfterTheLeaseTimeout() throws Exception {
+ List<Socket> accepted = new CopyOnWriteArrayList<>();
+ CountDownLatch firstConnected = new CountDownLatch(1);
+ try (ServerSocket server = new ServerSocket(0)) {
+ Thread acceptor = Thread.ofVirtual().start(() -> {
+ try {
+ while (true) {
+ accepted.add(server.accept());
+ firstConnected.countDown();
+ }
+ } catch (IOException expected) {
+ // The server socket closed at the end of the test.
+ }
+ });
+ try (FlociDockerHttpClient client = new FlociDockerHttpClient.Builder()
+ .dockerHost(URI.create("tcp://127.0.0.1:" + server.getLocalPort()))
+ .maxConnections(1)
+ .responseTimeout(Duration.ofSeconds(30))
+ .connectionRequestTimeout(Duration.ofSeconds(1))
+ .build()) {
+ // The one pooled connection is held by a request the server never answers.
+ Thread.ofVirtual().start(() -> {
+ try (DockerHttpClient.Response ignored = client.execute(get("/containers/held/wait"))) {
+ // Never reached: the server does not answer before the test ends.
+ } catch (RuntimeException expected) {
+ // The client closed when the test ended.
+ }
+ });
+ assertTrue(firstConnected.await(5, TimeUnit.SECONDS), "the first request should connect");
+
+ long start = System.nanoTime();
+ RuntimeException failure = assertThrows(RuntimeException.class,
+ () -> client.execute(get("/_ping")));
+ long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
+
+ assertTrue(failure.getCause() instanceof ConnectionRequestTimeoutException,
+ "a full pool should fail with ConnectionRequestTimeoutException, got " + failure.getCause());
+ assertTrue(elapsedMillis < 10_000,
+ "should fail near the 1 s lease timeout, took " + elapsedMillis + " ms");
+ } finally {
+ for (Socket socket : accepted) {
+ socket.close();
+ }
+ acceptor.interrupt();
+ }
+ }
+ }
+
+ private static DockerHttpClient.Request get(String path) {
+ return DockerHttpClient.Request.builder().method(DockerHttpClient.Request.Method.GET).path(path).build();
+ }
+}