Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(docker): retry transient registry failures during image pull (#1117)

Kenshin Okinaka44da9da6bb14
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java
index e5443bf232..c1ca757bb4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java
@@ -2,6 +2,8 @@ package io.github.hectorvent.floci.services.lambda.launcher;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.PullImageResultCallback;
+import com.github.dockerjava.api.exception.DockerClientException;
+import com.github.dockerjava.api.exception.InternalServerErrorException;
import com.github.dockerjava.api.model.AuthConfig;
import io.github.hectorvent.floci.config.EmulatorConfig;
import jakarta.enterprise.context.ApplicationScoped;
@@ -22,6 +24,9 @@ public class ImageCacheService {
private static final Logger LOG = Logger.getLogger(ImageCacheService.class);
+ static final int MAX_PULL_ATTEMPTS = 3;
+ static final long INITIAL_BACKOFF_MS = 500L;
+
private final DockerClient dockerClient;
private final List<EmulatorConfig.DockerConfig.RegistryCredential> registryCredentials;
private final Set<String> pulledImages = ConcurrentHashMap.newKeySet();
@@ -49,10 +54,11 @@ public class ImageCacheService {
}
LOG.infov("Pulling image: {0}", imageUri);
try {
- dockerClient.pullImageCmd(imageUri)
- .withAuthConfig(resolveAuth(imageUri))
- .exec(new PullImageResultCallback())
- .awaitCompletion(5, TimeUnit.MINUTES);
+ runWithRetry(imageUri, MAX_PULL_ATTEMPTS, INITIAL_BACKOFF_MS,
+ () -> dockerClient.pullImageCmd(imageUri)
+ .withAuthConfig(resolveAuth(imageUri))
+ .exec(new PullImageResultCallback())
+ .awaitCompletion(5, TimeUnit.MINUTES));
pulledImages.add(imageUri);
LOG.infov("Image pulled successfully: {0}", imageUri);
} catch (InterruptedException e) {
@@ -62,6 +68,56 @@ public class ImageCacheService {
}
}
+ /**
+ * Runs the given pull attempt, retrying on transient registry failures with exponential
+ * backoff. A failure is considered transient when either:
+ * <ul>
+ * <li>the docker daemon throws {@link InternalServerErrorException} directly (HTTP 500,
+ * e.g. ECR Public's {@code "toomanyrequests: Rate exceeded"}), or</li>
+ * <li>{@link PullImageResultCallback#awaitCompletion} rewraps a daemon error as
+ * {@link DockerClientException} with a message starting with
+ * {@code "Could not pull image: "} (the async-callback path; same root cause, just
+ * a different exception class).</li>
+ * </ul>
+ * Permanent failures (auth, missing image, malformed request, or any other
+ * {@code DockerClientException} not coming from the pull wrapper) keep surfacing their
+ * original docker-java exception subclass on the first attempt and are not retried.
+ */
+ static void runWithRetry(String imageUri, int maxAttempts, long initialBackoffMs,
+ PullAttempt attempt) throws InterruptedException {
+ long backoffMs = initialBackoffMs;
+ for (int i = 1; i <= maxAttempts; i++) {
+ try {
+ attempt.run();
+ return;
+ } catch (RuntimeException e) {
+ if (!isTransientPullFailure(e) || i == maxAttempts) {
+ throw e;
+ }
+ LOG.warnv(e, "Transient image pull failure for {0} (attempt {1}/{2}). "
+ + "Retrying in {3}ms.", imageUri, i, maxAttempts, backoffMs);
+ Thread.sleep(backoffMs);
+ backoffMs *= 2;
+ }
+ }
+ }
+
+ private static boolean isTransientPullFailure(RuntimeException e) {
+ if (e instanceof InternalServerErrorException) {
+ return true;
+ }
+ if (e instanceof DockerClientException && e.getMessage() != null
+ && e.getMessage().startsWith("Could not pull image: ")) {
+ return true;
+ }
+ return false;
+ }
+
+ @FunctionalInterface
+ interface PullAttempt {
+ void run() throws InterruptedException;
+ }
+
private boolean isLocalImagePresent(String imageUri) {
try {
dockerClient.inspectImageCmd(imageUri).exec();
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java
new file mode 100644
index 0000000000..1f3edba2b9
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java
@@ -0,0 +1,118 @@
+package io.github.hectorvent.floci.services.lambda.launcher;
+
+import com.github.dockerjava.api.exception.DockerClientException;
+import com.github.dockerjava.api.exception.DockerException;
+import com.github.dockerjava.api.exception.InternalServerErrorException;
+import com.github.dockerjava.api.exception.NotFoundException;
+import com.github.dockerjava.api.exception.UnauthorizedException;
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class ImageCacheServiceTest {
+
+ private static final String IMAGE = "public.ecr.aws/docker/library/alpine:latest";
+
+ @Test
+ void succeedsOnFirstAttempt() throws Exception {
+ AtomicInteger calls = new AtomicInteger();
+ ImageCacheService.runWithRetry(IMAGE, 3, 1L, calls::incrementAndGet);
+ assertEquals(1, calls.get());
+ }
+
+ @Test
+ void retriesOnTransient500AndSucceeds() throws Exception {
+ AtomicInteger calls = new AtomicInteger();
+ ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {
+ int attempt = calls.incrementAndGet();
+ if (attempt < 3) {
+ throw new InternalServerErrorException(
+ "Status 500: {\"message\":\"toomanyrequests: Rate exceeded\"}");
+ }
+ });
+ assertEquals(3, calls.get());
+ }
+
+ @Test
+ void exhaustsAttemptsAndRethrowsLast500() {
+ AtomicInteger calls = new AtomicInteger();
+ InternalServerErrorException ex = assertThrows(InternalServerErrorException.class,
+ () -> ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {
+ calls.incrementAndGet();
+ throw new InternalServerErrorException("backend unavailable");
+ }));
+ assertEquals(3, calls.get());
+ assertTrue(ex.getMessage().contains("backend unavailable"));
+ }
+
+ @Test
+ void doesNotRetryOnNotFound() {
+ AtomicInteger calls = new AtomicInteger();
+ assertThrows(NotFoundException.class,
+ () -> ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {
+ calls.incrementAndGet();
+ throw new NotFoundException("manifest unknown");
+ }));
+ assertEquals(1, calls.get());
+ }
+
+ @Test
+ void doesNotRetryOnUnauthorized() {
+ AtomicInteger calls = new AtomicInteger();
+ assertThrows(UnauthorizedException.class,
+ () -> ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {
+ calls.incrementAndGet();
+ throw new UnauthorizedException("denied");
+ }));
+ assertEquals(1, calls.get());
+ }
+
+ @Test
+ void doesNotRetryOnGenericDockerException() {
+ AtomicInteger calls = new AtomicInteger();
+ assertThrows(DockerException.class,
+ () -> ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {
+ calls.incrementAndGet();
+ throw new DockerException("connection refused", -1);
+ }));
+ assertEquals(1, calls.get());
+ }
+
+ @Test
+ void retriesOnPullWrapperDockerClientExceptionAndSucceeds() throws Exception {
+ AtomicInteger calls = new AtomicInteger();
+ ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {
+ if (calls.incrementAndGet() < 2) {
+ throw new DockerClientException(
+ "Could not pull image: toomanyrequests: Rate exceeded");
+ }
+ });
+ assertEquals(2, calls.get());
+ }
+
+ @Test
+ void doesNotRetryOnNonPullDockerClientException() {
+ AtomicInteger calls = new AtomicInteger();
+ assertThrows(DockerClientException.class,
+ () -> ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {
+ calls.incrementAndGet();
+ throw new DockerClientException("container start failed: exit 137");
+ }));
+ assertEquals(1, calls.get());
+ }
+
+ @Test
+ void propagatesInterrupted() {
+ AtomicInteger calls = new AtomicInteger();
+ assertThrows(InterruptedException.class,
+ () -> ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {
+ calls.incrementAndGet();
+ throw new InterruptedException("interrupted mid-pull");
+ }));
+ assertEquals(1, calls.get());
+ }
+}