Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(core): make the image pull timeout configurable (#6521)

Priamo Germosenb66ee9ecf0c8
diff --git a/docs/configuration/docker.md b/docs/configuration/docker.md
index cfbd281ef4..dbe3b7b6f7 100644
--- a/docs/configuration/docker.md
+++ b/docs/configuration/docker.md
@@ -92,6 +92,16 @@ floci:
Environment variable: `FLOCI_DOCKER_API_TIMEOUT_SECONDS`
+An image pull has a limit of its own: each attempt may take up to `image-pull-timeout-seconds`, 300 seconds (5 minutes) by default. A pull still downloading when the limit passes fails that attempt with "Could not pull image: Downloading" and is tried again, three attempts in all, so a launch that needs an image Docker does not have yet can wait up to about three times the limit before it fails; a pull that has reported no progress at all within the limit fails at once with "Timed out pulling image". A large image, such as an EMR Serverless runtime of 3.5 to 4.3 GB, can need longer on a slow connection: raise the limit, or pull the image with `docker pull` before the first launch. It must be at least 1:
+
+```yaml
+floci:
+ docker:
+ image-pull-timeout-seconds: 300
+```
+
+Environment variable: `FLOCI_DOCKER_IMAGE_PULL_TIMEOUT_SECONDS`
+
## Private Registry Authentication
Any service that pulls a container image from a private registry (Lambda image functions, custom OpenSearch images, private Postgres images, etc.) needs Docker credentials. Two approaches are supported and can be combined.
diff --git a/docs/services/emr-serverless.md b/docs/services/emr-serverless.md
index 98d3f5fc0e..48d542dfd6 100644
--- a/docs/services/emr-serverless.md
+++ b/docs/services/emr-serverless.md
@@ -107,9 +107,10 @@ publishes for customizing EMR Serverless images (`public.ecr.aws/emr-serverless/
`spark-submit` in local mode in it: the container holds the driver and its executors.
* **States**: a run is `SCHEDULED` while its image is pulled and its container starts (the first run
- of a release pulls its image, 3.5 to 4.3 GB unpacked; Floci gives a pull three attempts of five
- minutes, so on a slow connection run `docker pull` on the image first, or the run is `FAILED` with
- "Could not pull image"), and `RUNNING` once the Spark driver runs, which sets `startedAt`. It
+ of a release pulls its image, 3.5 to 4.3 GB unpacked; a pull still downloading after
+ [`floci.docker.image-pull-timeout-seconds`](../configuration/docker.md), five minutes by default,
+ is tried again, three attempts in all, so on a slow connection raise it or run `docker pull` on the
+ image first, or the run is `FAILED` with "Could not pull image"), and `RUNNING` once the Spark driver runs, which sets `startedAt`. It
is `SUCCESS` when the driver exits with code 0 and `FAILED` otherwise, with the exit code in
`stateDetails`. `CancelJobRun`, or a `BATCH` run passing its `executionTimeoutMinutes`, makes it
`CANCELLING` until the driver is stopped, then `CANCELLED`, as the job run states describe. A run
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index ef0080b5ca..35ae626bc8 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -3880,6 +3880,15 @@ public interface EmulatorConfig {
@WithDefault("300")
int apiTimeoutSeconds();
+ /**
+ * How long each attempt at pulling an image may take before it is given up, in seconds. A pull
+ * still downloading when the limit passes is tried again, three attempts in all, so a launch that
+ * needs an image can wait up to about three times this before it fails. A large image, such as an EMR Serverless runtime of 3.5 to 4.3 GB, can need longer
+ * on a slow connection.
+ */
+ @WithDefault("300")
+ int imagePullTimeoutSeconds();
+
/**
* Optional namespace inserted into Floci-managed child container and volume names.
* Useful when multiple Floci processes share one Docker daemon.
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.java
index 2563fbaafe..8176048964 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.java
@@ -407,6 +407,12 @@ public class DockerClientProducer {
return requireAtLeastOne(seconds, "floci.docker.api-timeout-seconds", "FLOCI_DOCKER_API_TIMEOUT_SECONDS");
}
+ /** At 0 every pull would be given up at once, so a launch that needs an image could never start. */
+ public static int validateImagePullTimeoutSeconds(int seconds) {
+ return requireAtLeastOne(seconds, "floci.docker.image-pull-timeout-seconds",
+ "FLOCI_DOCKER_IMAGE_PULL_TIMEOUT_SECONDS");
+ }
+
private static int requireAtLeastOne(int value, String setting, String envVar) {
if (value < 1) {
throw new IllegalArgumentException(setting + " (" + envVar + ") must be at least 1, got: " + value);
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java
index dcbe8628e5..d345072329 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java
@@ -11,6 +11,7 @@ import com.github.dockerjava.api.model.Info;
import com.github.dockerjava.api.model.PullResponseItem;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.config.EmulatorConfig.EcsServiceConfig.ImagePullBehavior;
+import io.github.hectorvent.floci.core.common.docker.DockerClientProducer;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
@@ -44,6 +45,8 @@ public class ImageCacheService {
private final DockerClient dockerClient;
private final List<EmulatorConfig.DockerConfig.RegistryCredential> registryCredentials;
+ // floci.docker.image-pull-timeout-seconds: how long each pull attempt may take.
+ private final int pullTimeoutSeconds;
private final Map<ImageKey, String> resolvedImages = new ConcurrentHashMap<>();
private final Set<ImageKey> pulledImages = ConcurrentHashMap.newKeySet();
private final ConcurrentHashMap<String, Object> locks = new ConcurrentHashMap<>();
@@ -53,6 +56,8 @@ public class ImageCacheService {
public ImageCacheService(DockerClient dockerClient, EmulatorConfig config) {
this.dockerClient = dockerClient;
this.registryCredentials = config.docker().registryCredentials();
+ this.pullTimeoutSeconds = DockerClientProducer.validateImagePullTimeoutSeconds(
+ config.docker().imagePullTimeoutSeconds());
}
public String ensureImageExists(String imageUri) {
@@ -186,8 +191,10 @@ public class ImageCacheService {
}
ManifestDigestCallback callback = new ManifestDigestCallback();
lastAttempt.set(callback);
- if (!pullImage.<PullImageResultCallback>exec(callback).awaitCompletion(5, TimeUnit.MINUTES)) {
- throw new DockerClientException("Timed out pulling image " + imageUri + " after 5 minutes");
+ if (!pullImage.<PullImageResultCallback>exec(callback)
+ .awaitCompletion(pullTimeoutSeconds, TimeUnit.SECONDS)) {
+ throw new DockerClientException("Timed out pulling image " + imageUri + " after "
+ + pullTimeoutSeconds + " seconds");
}
});
} catch (InterruptedException e) {
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index ff838a33d7..775fb4905b 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -377,6 +377,11 @@ floci:
# attempt: a control-plane request that is safe to repeat is tried up to six times. Keep in step with
# the @WithDefault on EmulatorConfig.DockerConfig.apiTimeoutSeconds: a value set here WINS over the annotation.
api-timeout-seconds: 300
+ # FLOCI_DOCKER_IMAGE_PULL_TIMEOUT_SECONDS: how long each attempt at an image pull may take; a pull
+ # still downloading when it passes is tried again, three attempts in all.
+ # Keep in step with the @WithDefault on EmulatorConfig.DockerConfig.imagePullTimeoutSeconds: a value
+ # set here WINS over the annotation.
+ image-pull-timeout-seconds: 300
resource-namespace: "" # FLOCI_DOCKER_RESOURCE_NAMESPACE; scopes child container/volume names
docker-config-path: ""
# extra-labels: # Extra Docker labels applied to every Floci-created container and volume
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.java
index 0408aed8b4..2063addb36 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.java
@@ -692,6 +692,22 @@ class DockerClientProducerTest {
assertEquals(300, DockerClientProducer.validateApiTimeoutSeconds(300));
}
+ // Catches: a zero pull timeout, which would give up on every pull at once.
+ @ParameterizedTest
+ @ValueSource(ints = {0, -1})
+ void validateImagePullTimeoutSeconds_belowOne_throwsNamingTheSetting(int seconds) {
+ IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> DockerClientProducer.validateImagePullTimeoutSeconds(seconds));
+
+ assertThat(e.getMessage(), containsString("floci.docker.image-pull-timeout-seconds"));
+ }
+
+ @Test
+ void validateImagePullTimeoutSeconds_positive_returnsItUnchanged() {
+ assertEquals(1, DockerClientProducer.validateImagePullTimeoutSeconds(1));
+ assertEquals(1800, DockerClientProducer.validateImagePullTimeoutSeconds(1800));
+ }
+
/**
* The pool size comes from {@code floci.docker.max-connections}. A followed log stream holds
* its connection until the container exits, and this fake daemon never answers, so with a
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java
index 2ecee9db43..c270079724 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java
@@ -32,6 +32,7 @@ import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Mockito.mock;
@@ -67,7 +68,28 @@ class ImageCacheServiceTest {
verify(pullImage).exec(any(PullImageResultCallback.class));
verify(pullImage, never()).withPlatform(nullable(String.class));
- verify(callback).awaitCompletion(5, TimeUnit.MINUTES);
+ verify(callback).awaitCompletion(300, TimeUnit.SECONDS);
+ }
+
+ @Test
+ void eachPullAttemptWaitsAsLongAsTheConfiguredImagePullTimeout() throws Exception {
+ DockerClient dockerClient = mock(DockerClient.class);
+ InspectImageCmd inspectImage = mock(InspectImageCmd.class);
+ PullImageCmd pullImage = mock(PullImageCmd.class);
+ PullImageResultCallback callback = completedPull();
+ when(dockerClient.inspectImageCmd(IMAGE)).thenReturn(inspectImage);
+ when(inspectImage.exec()).thenThrow(new NotFoundException("image not found"))
+ .thenReturn(new InspectImageResponse()
+ .withId("sha256:native")
+ .withOs("linux")
+ .withArch("amd64"));
+ when(dockerClient.pullImageCmd(IMAGE)).thenReturn(pullImage);
+ when(pullImage.withAuthConfig(any())).thenReturn(pullImage);
+ when(pullImage.exec(any(PullImageResultCallback.class))).thenReturn(callback);
+
+ newService(dockerClient, 1800).ensureImageExists(IMAGE);
+
+ verify(callback).awaitCompletion(1800, TimeUnit.SECONDS);
}
@Test
@@ -105,7 +127,7 @@ class ImageCacheServiceTest {
assertEquals("sha256:arm64", resolvedImage);
verify(pullImage).withPlatform("linux/arm64");
- verify(callback).awaitCompletion(5, TimeUnit.MINUTES);
+ verify(callback).awaitCompletion(300, TimeUnit.SECONDS);
}
@Test
@@ -697,11 +719,15 @@ class ImageCacheServiceTest {
private static PullImageResultCallback completedPull() throws InterruptedException {
PullImageResultCallback callback = mock(PullImageResultCallback.class);
- when(callback.awaitCompletion(5, TimeUnit.MINUTES)).thenReturn(true);
+ when(callback.awaitCompletion(anyLong(), any(TimeUnit.class))).thenReturn(true);
return callback;
}
private static ImageCacheService newService(DockerClient dockerClient) {
+ return newService(dockerClient, 300);
+ }
+
+ private static ImageCacheService newService(DockerClient dockerClient, int imagePullTimeoutSeconds) {
InfoCmd infoCmd = mock(InfoCmd.class);
when(dockerClient.infoCmd()).thenReturn(infoCmd);
when(infoCmd.exec()).thenReturn(new Info().withOsType("linux").withArchitecture("amd64"));
@@ -709,6 +735,7 @@ class ImageCacheServiceTest {
EmulatorConfig.DockerConfig dockerConfig = mock(EmulatorConfig.DockerConfig.class);
when(config.docker()).thenReturn(dockerConfig);
when(dockerConfig.registryCredentials()).thenReturn(List.of());
+ when(dockerConfig.imagePullTimeoutSeconds()).thenReturn(imagePullTimeoutSeconds);
return new ImageCacheService(dockerClient, config);
}
}