Light, fluffy, and always free - The AWS Local Emulator alternative
fix(docker): retry transient registry failures during image pull (#1117)
Kenshin Okinaka44da9da6bb14
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.javaindex e5443bf232..c1ca757bb4 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java@@ -2,6 +2,8 @@ package io.github.hectorvent.floci.services.lambda.launcher;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.command.PullImageResultCallback;+import com.github.dockerjava.api.exception.DockerClientException;+import com.github.dockerjava.api.exception.InternalServerErrorException;import com.github.dockerjava.api.model.AuthConfig;import io.github.hectorvent.floci.config.EmulatorConfig;import jakarta.enterprise.context.ApplicationScoped;@@ -22,6 +24,9 @@ public class ImageCacheService {private static final Logger LOG = Logger.getLogger(ImageCacheService.class);+ static final int MAX_PULL_ATTEMPTS = 3;+ static final long INITIAL_BACKOFF_MS = 500L;+private final DockerClient dockerClient;private final List<EmulatorConfig.DockerConfig.RegistryCredential> registryCredentials;private final Set<String> pulledImages = ConcurrentHashMap.newKeySet();@@ -49,10 +54,11 @@ public class ImageCacheService {}LOG.infov("Pulling image: {0}", imageUri);try {- dockerClient.pullImageCmd(imageUri)- .withAuthConfig(resolveAuth(imageUri))- .exec(new PullImageResultCallback())- .awaitCompletion(5, TimeUnit.MINUTES);+ runWithRetry(imageUri, MAX_PULL_ATTEMPTS, INITIAL_BACKOFF_MS,+ () -> dockerClient.pullImageCmd(imageUri)+ .withAuthConfig(resolveAuth(imageUri))+ .exec(new PullImageResultCallback())+ .awaitCompletion(5, TimeUnit.MINUTES));pulledImages.add(imageUri);LOG.infov("Image pulled successfully: {0}", imageUri);} catch (InterruptedException e) {@@ -62,6 +68,56 @@ public class ImageCacheService {}}+ /**+ * Runs the given pull attempt, retrying on transient registry failures with exponential+ * backoff. A failure is considered transient when either:+ * <ul>+ * <li>the docker daemon throws {@link InternalServerErrorException} directly (HTTP 500,+ * e.g. ECR Public's {@code "toomanyrequests: Rate exceeded"}), or</li>+ * <li>{@link PullImageResultCallback#awaitCompletion} rewraps a daemon error as+ * {@link DockerClientException} with a message starting with+ * {@code "Could not pull image: "} (the async-callback path; same root cause, just+ * a different exception class).</li>+ * </ul>+ * Permanent failures (auth, missing image, malformed request, or any other+ * {@code DockerClientException} not coming from the pull wrapper) keep surfacing their+ * original docker-java exception subclass on the first attempt and are not retried.+ */+ static void runWithRetry(String imageUri, int maxAttempts, long initialBackoffMs,+ PullAttempt attempt) throws InterruptedException {+ long backoffMs = initialBackoffMs;+ for (int i = 1; i <= maxAttempts; i++) {+ try {+ attempt.run();+ return;+ } catch (RuntimeException e) {+ if (!isTransientPullFailure(e) || i == maxAttempts) {+ throw e;+ }+ LOG.warnv(e, "Transient image pull failure for {0} (attempt {1}/{2}). "+ + "Retrying in {3}ms.", imageUri, i, maxAttempts, backoffMs);+ Thread.sleep(backoffMs);+ backoffMs *= 2;+ }+ }+ }++ private static boolean isTransientPullFailure(RuntimeException e) {+ if (e instanceof InternalServerErrorException) {+ return true;+ }+ if (e instanceof DockerClientException && e.getMessage() != null+ && e.getMessage().startsWith("Could not pull image: ")) {+ return true;+ }+ return false;+ }++ @FunctionalInterface+ interface PullAttempt {+ void run() throws InterruptedException;+ }+private boolean isLocalImagePresent(String imageUri) {try {dockerClient.inspectImageCmd(imageUri).exec();diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.javanew file mode 100644index 0000000000..1f3edba2b9--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java@@ -0,0 +1,118 @@+package io.github.hectorvent.floci.services.lambda.launcher;++import com.github.dockerjava.api.exception.DockerClientException;+import com.github.dockerjava.api.exception.DockerException;+import com.github.dockerjava.api.exception.InternalServerErrorException;+import com.github.dockerjava.api.exception.NotFoundException;+import com.github.dockerjava.api.exception.UnauthorizedException;+import org.junit.jupiter.api.Test;++import java.util.concurrent.atomic.AtomicInteger;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++class ImageCacheServiceTest {++ private static final String IMAGE = "public.ecr.aws/docker/library/alpine:latest";++ @Test+ void succeedsOnFirstAttempt() throws Exception {+ AtomicInteger calls = new AtomicInteger();+ ImageCacheService.runWithRetry(IMAGE, 3, 1L, calls::incrementAndGet);+ assertEquals(1, calls.get());+ }++ @Test+ void retriesOnTransient500AndSucceeds() throws Exception {+ AtomicInteger calls = new AtomicInteger();+ ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {+ int attempt = calls.incrementAndGet();+ if (attempt < 3) {+ throw new InternalServerErrorException(+ "Status 500: {\"message\":\"toomanyrequests: Rate exceeded\"}");+ }+ });+ assertEquals(3, calls.get());+ }++ @Test+ void exhaustsAttemptsAndRethrowsLast500() {+ AtomicInteger calls = new AtomicInteger();+ InternalServerErrorException ex = assertThrows(InternalServerErrorException.class,+ () -> ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {+ calls.incrementAndGet();+ throw new InternalServerErrorException("backend unavailable");+ }));+ assertEquals(3, calls.get());+ assertTrue(ex.getMessage().contains("backend unavailable"));+ }++ @Test+ void doesNotRetryOnNotFound() {+ AtomicInteger calls = new AtomicInteger();+ assertThrows(NotFoundException.class,+ () -> ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {+ calls.incrementAndGet();+ throw new NotFoundException("manifest unknown");+ }));+ assertEquals(1, calls.get());+ }++ @Test+ void doesNotRetryOnUnauthorized() {+ AtomicInteger calls = new AtomicInteger();+ assertThrows(UnauthorizedException.class,+ () -> ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {+ calls.incrementAndGet();+ throw new UnauthorizedException("denied");+ }));+ assertEquals(1, calls.get());+ }++ @Test+ void doesNotRetryOnGenericDockerException() {+ AtomicInteger calls = new AtomicInteger();+ assertThrows(DockerException.class,+ () -> ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {+ calls.incrementAndGet();+ throw new DockerException("connection refused", -1);+ }));+ assertEquals(1, calls.get());+ }++ @Test+ void retriesOnPullWrapperDockerClientExceptionAndSucceeds() throws Exception {+ AtomicInteger calls = new AtomicInteger();+ ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {+ if (calls.incrementAndGet() < 2) {+ throw new DockerClientException(+ "Could not pull image: toomanyrequests: Rate exceeded");+ }+ });+ assertEquals(2, calls.get());+ }++ @Test+ void doesNotRetryOnNonPullDockerClientException() {+ AtomicInteger calls = new AtomicInteger();+ assertThrows(DockerClientException.class,+ () -> ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {+ calls.incrementAndGet();+ throw new DockerClientException("container start failed: exit 137");+ }));+ assertEquals(1, calls.get());+ }++ @Test+ void propagatesInterrupted() {+ AtomicInteger calls = new AtomicInteger();+ assertThrows(InterruptedException.class,+ () -> ImageCacheService.runWithRetry(IMAGE, 3, 1L, () -> {+ calls.incrementAndGet();+ throw new InterruptedException("interrupted mid-pull");+ }));+ assertEquals(1, calls.get());+ }+}