Light, fluffy, and always free - The AWS Local Emulator alternative
feat(core): make the image pull timeout configurable (#6521)
Priamo Germosenb66ee9ecf0c8
diff --git a/docs/configuration/docker.md b/docs/configuration/docker.mdindex cfbd281ef4..dbe3b7b6f7 100644--- a/docs/configuration/docker.md+++ b/docs/configuration/docker.md@@ -92,6 +92,16 @@ floci:Environment variable: `FLOCI_DOCKER_API_TIMEOUT_SECONDS`+An image pull has a limit of its own: each attempt may take up to `image-pull-timeout-seconds`, 300 seconds (5 minutes) by default. A pull still downloading when the limit passes fails that attempt with "Could not pull image: Downloading" and is tried again, three attempts in all, so a launch that needs an image Docker does not have yet can wait up to about three times the limit before it fails; a pull that has reported no progress at all within the limit fails at once with "Timed out pulling image". A large image, such as an EMR Serverless runtime of 3.5 to 4.3 GB, can need longer on a slow connection: raise the limit, or pull the image with `docker pull` before the first launch. It must be at least 1:++```yaml+floci:+ docker:+ image-pull-timeout-seconds: 300+```++Environment variable: `FLOCI_DOCKER_IMAGE_PULL_TIMEOUT_SECONDS`+## Private Registry AuthenticationAny service that pulls a container image from a private registry (Lambda image functions, custom OpenSearch images, private Postgres images, etc.) needs Docker credentials. Two approaches are supported and can be combined.diff --git a/docs/services/emr-serverless.md b/docs/services/emr-serverless.mdindex 98d3f5fc0e..48d542dfd6 100644--- a/docs/services/emr-serverless.md+++ b/docs/services/emr-serverless.md@@ -107,9 +107,10 @@ publishes for customizing EMR Serverless images (`public.ecr.aws/emr-serverless/`spark-submit` in local mode in it: the container holds the driver and its executors.* **States**: a run is `SCHEDULED` while its image is pulled and its container starts (the first run- of a release pulls its image, 3.5 to 4.3 GB unpacked; Floci gives a pull three attempts of five- minutes, so on a slow connection run `docker pull` on the image first, or the run is `FAILED` with- "Could not pull image"), and `RUNNING` once the Spark driver runs, which sets `startedAt`. It+ of a release pulls its image, 3.5 to 4.3 GB unpacked; a pull still downloading after+ [`floci.docker.image-pull-timeout-seconds`](../configuration/docker.md), five minutes by default,+ is tried again, three attempts in all, so on a slow connection raise it or run `docker pull` on the+ image first, or the run is `FAILED` with "Could not pull image"), and `RUNNING` once the Spark driver runs, which sets `startedAt`. Itis `SUCCESS` when the driver exits with code 0 and `FAILED` otherwise, with the exit code in`stateDetails`. `CancelJobRun`, or a `BATCH` run passing its `executionTimeoutMinutes`, makes it`CANCELLING` until the driver is stopped, then `CANCELLED`, as the job run states describe. A rundiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex ef0080b5ca..35ae626bc8 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -3880,6 +3880,15 @@ public interface EmulatorConfig {@WithDefault("300")int apiTimeoutSeconds();+ /**+ * How long each attempt at pulling an image may take before it is given up, in seconds. A pull+ * still downloading when the limit passes is tried again, three attempts in all, so a launch that+ * needs an image can wait up to about three times this before it fails. A large image, such as an EMR Serverless runtime of 3.5 to 4.3 GB, can need longer+ * on a slow connection.+ */+ @WithDefault("300")+ int imagePullTimeoutSeconds();+/*** Optional namespace inserted into Floci-managed child container and volume names.* Useful when multiple Floci processes share one Docker daemon.diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.javaindex 2563fbaafe..8176048964 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducer.java@@ -407,6 +407,12 @@ public class DockerClientProducer {return requireAtLeastOne(seconds, "floci.docker.api-timeout-seconds", "FLOCI_DOCKER_API_TIMEOUT_SECONDS");}+ /** At 0 every pull would be given up at once, so a launch that needs an image could never start. */+ public static int validateImagePullTimeoutSeconds(int seconds) {+ return requireAtLeastOne(seconds, "floci.docker.image-pull-timeout-seconds",+ "FLOCI_DOCKER_IMAGE_PULL_TIMEOUT_SECONDS");+ }+private static int requireAtLeastOne(int value, String setting, String envVar) {if (value < 1) {throw new IllegalArgumentException(setting + " (" + envVar + ") must be at least 1, got: " + value);diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.javaindex dcbe8628e5..d345072329 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java@@ -11,6 +11,7 @@ import com.github.dockerjava.api.model.Info;import com.github.dockerjava.api.model.PullResponseItem;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.config.EmulatorConfig.EcsServiceConfig.ImagePullBehavior;+import io.github.hectorvent.floci.core.common.docker.DockerClientProducer;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -44,6 +45,8 @@ public class ImageCacheService {private final DockerClient dockerClient;private final List<EmulatorConfig.DockerConfig.RegistryCredential> registryCredentials;+ // floci.docker.image-pull-timeout-seconds: how long each pull attempt may take.+ private final int pullTimeoutSeconds;private final Map<ImageKey, String> resolvedImages = new ConcurrentHashMap<>();private final Set<ImageKey> pulledImages = ConcurrentHashMap.newKeySet();private final ConcurrentHashMap<String, Object> locks = new ConcurrentHashMap<>();@@ -53,6 +56,8 @@ public class ImageCacheService {public ImageCacheService(DockerClient dockerClient, EmulatorConfig config) {this.dockerClient = dockerClient;this.registryCredentials = config.docker().registryCredentials();+ this.pullTimeoutSeconds = DockerClientProducer.validateImagePullTimeoutSeconds(+ config.docker().imagePullTimeoutSeconds());}public String ensureImageExists(String imageUri) {@@ -186,8 +191,10 @@ public class ImageCacheService {}ManifestDigestCallback callback = new ManifestDigestCallback();lastAttempt.set(callback);- if (!pullImage.<PullImageResultCallback>exec(callback).awaitCompletion(5, TimeUnit.MINUTES)) {- throw new DockerClientException("Timed out pulling image " + imageUri + " after 5 minutes");+ if (!pullImage.<PullImageResultCallback>exec(callback)+ .awaitCompletion(pullTimeoutSeconds, TimeUnit.SECONDS)) {+ throw new DockerClientException("Timed out pulling image " + imageUri + " after "+ + pullTimeoutSeconds + " seconds");}});} catch (InterruptedException e) {diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex ff838a33d7..775fb4905b 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -377,6 +377,11 @@ floci:# attempt: a control-plane request that is safe to repeat is tried up to six times. Keep in step with# the @WithDefault on EmulatorConfig.DockerConfig.apiTimeoutSeconds: a value set here WINS over the annotation.api-timeout-seconds: 300+ # FLOCI_DOCKER_IMAGE_PULL_TIMEOUT_SECONDS: how long each attempt at an image pull may take; a pull+ # still downloading when it passes is tried again, three attempts in all.+ # Keep in step with the @WithDefault on EmulatorConfig.DockerConfig.imagePullTimeoutSeconds: a value+ # set here WINS over the annotation.+ image-pull-timeout-seconds: 300resource-namespace: "" # FLOCI_DOCKER_RESOURCE_NAMESPACE; scopes child container/volume namesdocker-config-path: ""# extra-labels: # Extra Docker labels applied to every Floci-created container and volumediff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.javaindex 0408aed8b4..2063addb36 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/DockerClientProducerTest.java@@ -692,6 +692,22 @@ class DockerClientProducerTest {assertEquals(300, DockerClientProducer.validateApiTimeoutSeconds(300));}+ // Catches: a zero pull timeout, which would give up on every pull at once.+ @ParameterizedTest+ @ValueSource(ints = {0, -1})+ void validateImagePullTimeoutSeconds_belowOne_throwsNamingTheSetting(int seconds) {+ IllegalArgumentException e = assertThrows(IllegalArgumentException.class,+ () -> DockerClientProducer.validateImagePullTimeoutSeconds(seconds));++ assertThat(e.getMessage(), containsString("floci.docker.image-pull-timeout-seconds"));+ }++ @Test+ void validateImagePullTimeoutSeconds_positive_returnsItUnchanged() {+ assertEquals(1, DockerClientProducer.validateImagePullTimeoutSeconds(1));+ assertEquals(1800, DockerClientProducer.validateImagePullTimeoutSeconds(1800));+ }+/*** The pool size comes from {@code floci.docker.max-connections}. A followed log stream holds* its connection until the container exits, and this fake daemon never answers, so with adiff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.javaindex 2ecee9db43..c270079724 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java@@ -32,6 +32,7 @@ import static org.junit.jupiter.api.Assertions.assertSame;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyLong;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.nullable;import static org.mockito.Mockito.mock;@@ -67,7 +68,28 @@ class ImageCacheServiceTest {verify(pullImage).exec(any(PullImageResultCallback.class));verify(pullImage, never()).withPlatform(nullable(String.class));- verify(callback).awaitCompletion(5, TimeUnit.MINUTES);+ verify(callback).awaitCompletion(300, TimeUnit.SECONDS);+ }++ @Test+ void eachPullAttemptWaitsAsLongAsTheConfiguredImagePullTimeout() throws Exception {+ DockerClient dockerClient = mock(DockerClient.class);+ InspectImageCmd inspectImage = mock(InspectImageCmd.class);+ PullImageCmd pullImage = mock(PullImageCmd.class);+ PullImageResultCallback callback = completedPull();+ when(dockerClient.inspectImageCmd(IMAGE)).thenReturn(inspectImage);+ when(inspectImage.exec()).thenThrow(new NotFoundException("image not found"))+ .thenReturn(new InspectImageResponse()+ .withId("sha256:native")+ .withOs("linux")+ .withArch("amd64"));+ when(dockerClient.pullImageCmd(IMAGE)).thenReturn(pullImage);+ when(pullImage.withAuthConfig(any())).thenReturn(pullImage);+ when(pullImage.exec(any(PullImageResultCallback.class))).thenReturn(callback);++ newService(dockerClient, 1800).ensureImageExists(IMAGE);++ verify(callback).awaitCompletion(1800, TimeUnit.SECONDS);}@Test@@ -105,7 +127,7 @@ class ImageCacheServiceTest {assertEquals("sha256:arm64", resolvedImage);verify(pullImage).withPlatform("linux/arm64");- verify(callback).awaitCompletion(5, TimeUnit.MINUTES);+ verify(callback).awaitCompletion(300, TimeUnit.SECONDS);}@Test@@ -697,11 +719,15 @@ class ImageCacheServiceTest {private static PullImageResultCallback completedPull() throws InterruptedException {PullImageResultCallback callback = mock(PullImageResultCallback.class);- when(callback.awaitCompletion(5, TimeUnit.MINUTES)).thenReturn(true);+ when(callback.awaitCompletion(anyLong(), any(TimeUnit.class))).thenReturn(true);return callback;}private static ImageCacheService newService(DockerClient dockerClient) {+ return newService(dockerClient, 300);+ }++ private static ImageCacheService newService(DockerClient dockerClient, int imagePullTimeoutSeconds) {InfoCmd infoCmd = mock(InfoCmd.class);when(dockerClient.infoCmd()).thenReturn(infoCmd);when(infoCmd.exec()).thenReturn(new Info().withOsType("linux").withArchitecture("amd64"));@@ -709,6 +735,7 @@ class ImageCacheServiceTest {EmulatorConfig.DockerConfig dockerConfig = mock(EmulatorConfig.DockerConfig.class);when(config.docker()).thenReturn(dockerConfig);when(dockerConfig.registryCredentials()).thenReturn(List.of());+ when(dockerConfig.imagePullTimeoutSeconds()).thenReturn(imagePullTimeoutSeconds);return new ImageCacheService(dockerClient, config);}}