Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ecs): pull task images at launch so a moved tag reaches the next task (#4583)

Abhigyad73e33fe4bf0
diff --git a/.github/ci/prefetch-images.sh b/.github/ci/prefetch-images.sh
index d4ecc99586..ac63b6b10c 100755
--- a/.github/ci/prefetch-images.sh
+++ b/.github/ci/prefetch-images.sh
@@ -66,9 +66,13 @@ BUSYBOX_IMAGE="$(test_image BUSYBOX)"
starts ContainerHostNetworkDockerIntegrationTest EcsServiceDiscoveryDockerIntegrationTest \
EcsContainerManagerEfsIsolationDockerIntegrationTest EcsContainerManagerFirelensDockerIntegrationTest \
EcsContainerManagerStatsDockerIntegrationTest EcsContainerManagerVolumesFromDockerIntegrationTest \
- EcsExecChannelDockerIntegrationTest EcsLeftoverContainersDockerIntegrationTest BatchDockerRunnerDockerIntegrationTest \
- SageMakerDockerIntegrationTest \
+ EcsExecChannelDockerIntegrationTest EcsImagePullDockerIntegrationTest EcsLeftoverContainersDockerIntegrationTest \
+ BatchDockerRunnerDockerIntegrationTest SageMakerDockerIntegrationTest \
&& [ -n "$BUSYBOX_IMAGE" ] && pull "$BUSYBOX_IMAGE"
+# EcsImagePullDockerIntegrationTest runs its own registry to move a tag in; the /ecr/ line below
+# covers the shards that run the ECR tests.
+REGISTRY_IMAGE="$(test_image REGISTRY)"
+starts EcsImagePullDockerIntegrationTest && [ -n "$REGISTRY_IMAGE" ] && pull "$REGISTRY_IMAGE"
PYTHON_ALPINE_IMAGE="$(test_image PYTHON_ALPINE)"
starts ContainerCaBundleDockerIntegrationTest EcsCredentialsProxyDockerIntegrationTest SageMakerDockerIntegrationTest \
&& [ -n "$PYTHON_ALPINE_IMAGE" ] && pull "$PYTHON_ALPINE_IMAGE"
diff --git a/docs/configuration/advanced/application-yml.md b/docs/configuration/advanced/application-yml.md
index 46677f64b3..0f74feffe5 100644
--- a/docs/configuration/advanced/application-yml.md
+++ b/docs/configuration/advanced/application-yml.md
@@ -279,6 +279,7 @@ floci:
enabled: true
mock: false # true = tasks go to RUNNING without Docker (useful for CI)
docker-network: floci-net # required for task-role credentials; must be user-defined
+ image-pull-behavior: default # as ECS_IMAGE_PULL_BEHAVIOR: default | always | once | prefer-cached
task-role-credentials:
enabled: false # vend real task IAM role credentials to task containers
ttl-seconds: 21600 # six hours, matching AWS
@@ -341,6 +342,7 @@ All keys in this table are declared on `EmulatorConfig` and accept environment v
| `FLOCI_SERVICES_ECS_HOST_VOLUME_ROOTS` | *(unset)* | Comma-separated allowlist of parent directories for host volume bind mounts; by default (unset, and `ALLOW_UNSAFE_HOST_VOLUMES=false`) every host volume `sourcePath` is rejected |
| `FLOCI_SERVICES_ECS_ALLOW_UNSAFE_HOST_VOLUMES` | `false` | Allow any host path, bypassing the host-volume-roots allowlist (traversal, bare root, and the Docker socket or an ancestor directory of it are still always rejected) |
| `FLOCI_SERVICES_ECS_RECONCILE_CONTAINERS_ON_STARTUP` | `true` | Remove the ECS containers a previous run of this Floci left on the daemon before replacement tasks start |
+| `FLOCI_SERVICES_ECS_IMAGE_PULL_BEHAVIOR` | `default` | How task images are pulled at launch, as the ECS agent's `ECS_IMAGE_PULL_BEHAVIOR`: `default`, `always`, `once` or `prefer-cached` |
| `FLOCI_SERVICES_IAM_ENFORCEMENT_ENABLED` | `false` | Enforce IAM identity-based policies on every request when `true` |
| `FLOCI_SERVICES_OPENSEARCH_MOCK` | `false` | Skip Docker; domains appear active immediately (useful for CI) |
| `FLOCI_SERVICES_OPENSEARCH_KEEP_RUNNING_ON_SHUTDOWN` | `false` | Leave OpenSearch containers running after Floci stops |
diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md
index 0f3543b1d5..1037c54177 100644
--- a/docs/configuration/environment-variables.md
+++ b/docs/configuration/environment-variables.md
@@ -506,6 +506,7 @@ These services spawn Docker containers. They require access to the Docker socket
| `FLOCI_SERVICES_ECS_HOST_VOLUME_ROOTS` | _(none)_ | Comma-separated allowlist of parent directories host volume `sourcePath`s must resolve under. By default (unset, and `ALLOW_UNSAFE_HOST_VOLUMES=false`) every host volume `sourcePath` is rejected |
| `FLOCI_SERVICES_ECS_ALLOW_UNSAFE_HOST_VOLUMES` | `false` | Allow any host path, bypassing `HOST_VOLUME_ROOTS`; traversal, the bare root, and the Docker socket (or an ancestor directory of it, e.g. `/var/run`) are still always rejected |
| `FLOCI_SERVICES_ECS_RECONCILE_CONTAINERS_ON_STARTUP` | `true` | On startup, in Docker mode, remove the ECS containers a previous run of this Floci left on the daemon (matched by resource namespace and API port) before replacement tasks start |
+| `FLOCI_SERVICES_ECS_IMAGE_PULL_BEHAVIOR` | `default` | How task images are pulled at launch, with the values and semantics of the ECS agent's `ECS_IMAGE_PULL_BEHAVIOR`: `default` pulls every launch and falls back to the cached image, `always` fails the task when the pull fails, `once` pulls once per Floci process, `prefer-cached` pulls only when nothing is cached |
| `FLOCI_SERVICES_ECS_TASK_ROLE_CREDENTIALS_ENABLED` | `false` | Vend real task IAM role credentials to task containers over the AWS container-credentials contract. Requires `FLOCI_SERVICES_ECS_DOCKER_NETWORK` to name a user-defined Docker network: the default bridge will not work |
| `FLOCI_SERVICES_ECS_TASK_ROLE_CREDENTIALS_TTL_SECONDS` | `21600` | Lifetime of the vended credentials. Matches the six hours AWS documents for task-role credentials |
| `FLOCI_SERVICES_ECS_TASK_ROLE_CREDENTIALS_PORT` | `51679` | Port on the Floci host serving the credentials endpoint. Not the address task containers use, which is always `169.254.170.2:80` |
diff --git a/docs/services/ecs.md b/docs/services/ecs.md
index d4522a62ff..e7149a3321 100644
--- a/docs/services/ecs.md
+++ b/docs/services/ecs.md
@@ -758,11 +758,40 @@ Every `awsvpc` task receives an ENI in its subnet. With `FLOCI_NETWORK_SECURITY_
| `FLOCI_SERVICES_ECS_HOST_VOLUME_ROOTS` | *(unset)* | Approved parent directories for host volume bind mounts (`volumes[].host.sourcePath`) |
| `FLOCI_SERVICES_ECS_ALLOW_UNSAFE_HOST_VOLUMES` | `false` | Allow any host path, bypassing the `HOST_VOLUME_ROOTS` allowlist; traversal, the bare root, and the Docker socket are still always rejected |
| `FLOCI_SERVICES_ECS_RECONCILE_CONTAINERS_ON_STARTUP` | `true` | On startup, in Docker mode, remove the ECS containers a previous run of this Floci left on the daemon before the scheduler starts replacement tasks (see [Restarts](#restarts)) |
+| `FLOCI_SERVICES_ECS_IMAGE_PULL_BEHAVIOR` | `default` | How task images are pulled at launch, as the ECS agent's `ECS_IMAGE_PULL_BEHAVIOR`: `default`, `always`, `once` or `prefer-cached` (see [Image pulls](#image-pulls)) |
| `FLOCI_SERVICES_ECS_TASK_ROLE_CREDENTIALS_ENABLED` | `false` | Vend real task IAM role credentials to task containers |
| `FLOCI_SERVICES_ECS_TASK_ROLE_CREDENTIALS_TTL_SECONDS` | `21600` | Lifetime of the vended credentials |
| `FLOCI_SERVICES_ECS_TASK_ROLE_CREDENTIALS_PORT` | `51679` | Floci-side port serving the credentials endpoint |
| `FLOCI_SERVICES_ECS_TASK_ROLE_CREDENTIALS_PROXY_IMAGE` | `floci/network-helper:local` | Image for the per-network credentials proxy |
+### Image pulls
+
+Floci pulls a task's container images when the task starts, before it creates any of its
+containers, the way the ECS agent does. `FLOCI_SERVICES_ECS_IMAGE_PULL_BEHAVIOR` takes the values
+of the agent's `ECS_IMAGE_PULL_BEHAVIOR`:
+
+| Value | Behavior |
+|---|---|
+| `default` | Pull on every launch. When the pull fails, run the cached image if there is one, otherwise the task fails. |
+| `always` | Pull on every launch. When the pull fails, the task fails. |
+| `once` | Pull when Floci has not pulled the image since it started, or the cached image is gone. Otherwise run the cached image. |
+| `prefer-cached` | Pull only when there is no cached image. |
+
+Under `default` and `always`, a tag moved in its registry (a rebuilt `:latest`) is what the next
+task runs, so `UpdateService --force-new-deployment` picks it up. `default` also keeps a locally
+built image that was never pushed anywhere working: its pull fails and the cached image runs,
+with a warning in the log. Whichever behavior applies, the cached image is the one the local tag
+names at launch, so a `docker pull` or `docker build` on the Docker host is also picked up.
+
+Each container reports the digest of the manifest it runs as `imageDigest` in `DescribeTasks`
+(and `ImageID` in the task metadata endpoint): the digest the registry reported when the task's
+launch pulled the tag, or, for a launch that runs the cached image, the image's digest for that
+repository. An image that has never been pulled from or pushed to the repository its reference
+names, such as one built locally, has no manifest digest and reports none, as does a cached image
+holding several digests for the repository when nothing local says which one the tag names. A task definition
+image given as a bare image id (`sha256:...`) runs that local image without a pull. Floci does not pin a service's digests across the tasks of one deployment, so a tag
+moved while a service is scaling can put tasks of the same deployment on different images.
+
### Task IAM role credentials
Off by default. With `FLOCI_SERVICES_ECS_TASK_ROLE_CREDENTIALS_ENABLED=true`, a task whose
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 3872d59942..543d1273d5 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -2122,7 +2122,30 @@ public interface EmulatorConfig {
@WithDefault("true")
boolean reconcileContainersOnStartup();
+ /**
+ * How a task's container images are pulled when the task starts, with the values and
+ * semantics of the ECS agent's {@code ECS_IMAGE_PULL_BEHAVIOR}. The default pulls on every
+ * launch, so a tag moved in its registry (a rebuilt {@code :latest}) is what the next task
+ * runs.
+ */
+ @WithDefault("default")
+ ImagePullBehavior imagePullBehavior();
+
EcsTaskRoleCredentialsConfig taskRoleCredentials();
+
+ enum ImagePullBehavior {
+ /** Pull on every launch; when the pull fails, run the cached image if there is one. */
+ DEFAULT,
+ /** Pull on every launch; when the pull fails, the task fails. */
+ ALWAYS,
+ /**
+ * Pull when Floci has not pulled the image since it started or the cached image is
+ * gone; otherwise run the cached image.
+ */
+ ONCE,
+ /** Pull only when there is no cached image. */
+ PREFER_CACHED
+ }
}
interface EcsTaskRoleCredentialsConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.java
index 32061a6a4e..d291f7f418 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.java
@@ -1,7 +1,5 @@
package io.github.hectorvent.floci.core.common.docker;
-import io.github.hectorvent.floci.config.EmulatorConfig;
-import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
import com.github.dockerjava.api.model.AccessMode;
import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.DeviceRequest;
@@ -10,6 +8,9 @@ import com.github.dockerjava.api.model.Mount;
import com.github.dockerjava.api.model.MountType;
import com.github.dockerjava.api.model.Volume;
import com.github.dockerjava.api.model.VolumesFrom;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
@@ -92,7 +93,7 @@ public class ContainerBuilder {
}
static String resolveImage(String image, Optional<String> imageRegistryBase) {
- if (image == null || image.isBlank()) {
+ if (image == null || image.isBlank() || ImageCacheService.isImageId(image)) {
return image;
}
return normalizeImageRegistryBase(imageRegistryBase)
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index 4f08114414..1ae57ceb43 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -1,8 +1,5 @@
package io.github.hectorvent.floci.core.common.docker;
-import io.github.hectorvent.floci.config.ContainerCaBundle;
-import io.github.hectorvent.floci.config.EmulatorConfig;
-import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.CreateContainerCmd;
@@ -22,6 +19,11 @@ import com.github.dockerjava.api.model.Mount;
import com.github.dockerjava.api.model.MountType;
import com.github.dockerjava.api.model.Ports;
import com.github.dockerjava.core.command.WaitContainerResultCallback;
+import io.github.hectorvent.floci.config.ContainerCaBundle;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.config.EmulatorConfig.EcsServiceConfig.ImagePullBehavior;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;
import io.quarkus.runtime.annotations.RegisterForReflection;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
@@ -114,6 +116,15 @@ public class ContainerLifecycleManager {
}
}
+ /**
+ * Resolves the image a container launch is about to run, pulling it as the ECS agent does
+ * under {@code behavior}. A spec built from the returned image id runs exactly that image,
+ * whatever a later pull of the same reference moves its tag to.
+ */
+ public LaunchImage resolveImageForLaunch(String image, ImagePullBehavior behavior) {
+ return imageCacheService.resolveForLaunch(image, behavior);
+ }
+
/**
* Creates a container without starting it. Use {@link #startCreated} to
* start it after any pre-start setup (e.g. copying files into the
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
index 31fc92a742..1344dbde8a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
@@ -38,6 +38,7 @@ import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
import io.github.hectorvent.floci.services.ecs.model.TaskNetworkInterface;
import io.github.hectorvent.floci.services.ecs.model.Volume;
import io.github.hectorvent.floci.services.ecs.model.VolumeFrom;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
@@ -249,6 +250,7 @@ public class EcsContainerManager {
Map<ContainerDefinition, List<String>> envVarsByContainer = new LinkedHashMap<>();
// Resolved before any container is created, so a registry-startup failure can't leak one already started.
Map<ContainerDefinition, String> imagesByContainer = new LinkedHashMap<>();
+ Map<String, String> imageDigestsByContainer = new LinkedHashMap<>();
// The task metadata id has to exist before the container does: its own environment carries
// the URI, so it cannot be derived from the Docker id the daemon hands back afterwards.
Map<String, String> metadataIdsByContainer = new LinkedHashMap<>();
@@ -262,7 +264,20 @@ public class EcsContainerManager {
metadataIdsByContainer.put(def.getName(), metadataId);
envVarsByContainer.put(def, buildEnvVars(def, overridesByName.get(def.getName()), region,
metadataId, taskRoleEndpoint.vending()));
- imagesByContainer.put(def, ecrRegistryManager.rewriteImageUri(def.getImage()));
+ imagesByContainer.put(def,
+ containerBuilder.resolveImage(ecrRegistryManager.rewriteImageUri(def.getImage())));
+ }
+ // Pulled for every container before any is created, as the ECS agent does, so a tag
+ // moved in its registry since the last launch is what this task runs. Each container is
+ // created from the image id its pull resolved to, so a concurrent launch that moves the
+ // tag again cannot swap the image under this task's reported digest.
+ for (ContainerDefinition def : launchOrder) {
+ LaunchImage launchImage = lifecycleManager.resolveImageForLaunch(imagesByContainer.get(def),
+ config.services().ecs().imagePullBehavior());
+ imagesByContainer.put(def, launchImage.imageId());
+ if (launchImage.manifestDigest() != null) {
+ imageDigestsByContainer.put(def.getName(), launchImage.manifestDigest());
+ }
}
if (firelensRouter != null) {
@@ -489,6 +504,7 @@ public class EcsContainerManager {
// Build ECS container model
Container container = buildContainer(task.getTaskArn(), def, dockerId, networkBindings, region,
metadataIdsByContainer.get(def.getName()));
+ container.setImageDigest(imageDigestsByContainer.get(def.getName()));
runtimeContainers.add(container);
containerIds.put(def.getName(), dockerId);
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java
index 6a7bb0aad9..1db8d8e644 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java
@@ -8,19 +8,27 @@ import com.github.dockerjava.api.exception.DockerClientException;
import com.github.dockerjava.api.exception.InternalServerErrorException;
import com.github.dockerjava.api.model.AuthConfig;
import com.github.dockerjava.api.model.Info;
+import com.github.dockerjava.api.model.PullResponseItem;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.config.EmulatorConfig.EcsServiceConfig.ImagePullBehavior;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
/**
- * Ensures each Docker image is pulled only once per platform.
+ * Ensures each Docker image is pulled only once per platform, except for launches that ask
+ * for the ECS agent's pull behaviour through {@link #resolveForLaunch}.
* Thread-safe using ConcurrentHashMap for double-checked locking per image.
*/
@ApplicationScoped
@@ -30,10 +38,14 @@ public class ImageCacheService {
static final int MAX_PULL_ATTEMPTS = 3;
static final long INITIAL_BACKOFF_MS = 500L;
+ private static final List<String> DOCKER_HUB_PREFIXES = List.of("docker.io/", "index.docker.io/", "library/");
+ private static final Pattern IMAGE_ID = Pattern.compile("sha256:[0-9a-f]{64}");
+ private static final String DIGEST_STATUS = "Digest: ";
private final DockerClient dockerClient;
private final List<EmulatorConfig.DockerConfig.RegistryCredential> registryCredentials;
private final Map<ImageKey, String> resolvedImages = new ConcurrentHashMap<>();
+ private final Set<ImageKey> pulledImages = ConcurrentHashMap.newKeySet();
private final ConcurrentHashMap<String, Object> locks = new ConcurrentHashMap<>();
private volatile String daemonPlatform;
@@ -48,6 +60,9 @@ public class ImageCacheService {
}
public String ensureImageExists(String imageUri, String platform) {
+ if (isImageId(imageUri)) {
+ return existingImageId(imageUri);
+ }
boolean explicitPlatform = platform != null && !platform.isBlank();
String requestedPlatform = explicitPlatform ? platform.trim() : daemonPlatform();
ImageKey imageKey = new ImageKey(imageUri, requestedPlatform);
@@ -68,26 +83,174 @@ public class ImageCacheService {
LOG.infov("Image already present locally, skipping pull: {0}", imageUri);
return resolvedImage;
}
- LOG.infov("Pulling image: {0}", imageUri);
- try {
- runWithRetry(imageUri, MAX_PULL_ATTEMPTS, INITIAL_BACKOFF_MS, () -> {
- PullImageCmd pullImage = dockerClient.pullImageCmd(imageUri)
- .withAuthConfig(resolveAuth(imageUri));
- if (explicitPlatform) {
- pullImage.withPlatform(requestedPlatform);
- }
- pullImage.exec(new PullImageResultCallback())
- .awaitCompletion(5, TimeUnit.MINUTES);
- });
- resolvedImage = resolvedImageReference(imageUri, inspectLocalImage(imageUri));
- resolvedImages.put(imageKey, resolvedImage);
- LOG.infov("Image pulled successfully: {0}", imageUri);
- return resolvedImage;
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new RuntimeException("Interrupted while pulling image: " + imageUri, e);
+ resolvedImage = resolvedImageReference(imageUri,
+ pull(imageKey, explicitPlatform ? requestedPlatform : null).image());
+ resolvedImages.put(imageKey, resolvedImage);
+ return resolvedImage;
+ }
+ }
+
+ /**
+ * Resolves the image a container launch runs, pulling it as the ECS agent does under
+ * {@code ECS_IMAGE_PULL_BEHAVIOR}.
+ *
+ * <p>Unlike {@link #ensureImageExists}, this never reuses the id a reference named the first
+ * time it was seen: the local tag is read again on every call, so a tag moved by a pull (this
+ * one or anyone else's) is what the next launch runs. The launch creates its container from the
+ * returned image id rather than from the reference, so a pull by an overlapping launch that
+ * moves the tag again cannot change what this one runs.
+ */
+ public LaunchImage resolveForLaunch(String imageUri, ImagePullBehavior behavior) {
+ if (isImageId(imageUri)) {
+ return new LaunchImage(existingImageId(imageUri), null);
+ }
+ ImageKey imageKey = new ImageKey(imageUri, daemonPlatform());
+ Object lock = locks.computeIfAbsent(imageUri, k -> new Object());
+ synchronized (lock) {
+ LocalImage image = switch (behavior) {
+ case DEFAULT -> pullOrUseCached(imageKey);
+ case ALWAYS -> pull(imageKey, null);
+ case ONCE -> pulledImages.contains(imageKey) ? cachedOrPull(imageKey) : pull(imageKey, null);
+ case PREFER_CACHED -> cachedOrPull(imageKey);
+ };
+ String imageId = resolvedImageReference(imageUri, image.image());
+ resolvedImages.put(imageKey, imageId);
+ String digest = image.pulledDigest() != null && imageUri.indexOf('@') < 0
+ ? image.pulledDigest()
+ : manifestDigest(imageUri, image.image()).orElse(null);
+ return new LaunchImage(imageId, digest);
+ }
+ }
+
+ /**
+ * The image a launch resolved a reference to.
+ *
+ * @param imageId the immutable id of the image the launch runs
+ * @param manifestDigest the image's manifest digest in the repository it was named by, null when
+ * the image was never pulled from or pushed to that repository (a locally built image)
+ */
+ public record LaunchImage(String imageId, String manifestDigest) {}
+
+ /** Whether a reference is an image id, which names one immutable image and never needs a pull. */
+ public static boolean isImageId(String reference) {
+ return reference != null && IMAGE_ID.matcher(reference).matches();
+ }
+
+ private String existingImageId(String imageId) {
+ InspectImageResponse image = inspectLocalImage(imageId);
+ if (image == null) {
+ throw new DockerClientException("Image no longer exists: " + imageId);
+ }
+ return resolvedImageReference(imageId, image);
+ }
+
+ private LocalImage pullOrUseCached(ImageKey imageKey) {
+ try {
+ return pull(imageKey, null);
+ } catch (RuntimeException e) {
+ InspectImageResponse cached = Thread.currentThread().isInterrupted() ? null : cachedImage(imageKey);
+ if (cached == null) {
+ throw e;
+ }
+ LOG.warnv("Could not pull image {0}, using the cached image {1}: {2}",
+ imageKey.imageUri(), cached.getId(), e.getMessage());
+ return new LocalImage(cached, null);
+ }
+ }
+
+ private LocalImage cachedOrPull(ImageKey imageKey) {
+ InspectImageResponse cached = cachedImage(imageKey);
+ return cached != null ? new LocalImage(cached, null) : pull(imageKey, null);
+ }
+
+ private InspectImageResponse cachedImage(ImageKey imageKey) {
+ InspectImageResponse local = inspectLocalImage(imageKey.imageUri());
+ return matchesPlatform(local, imageKey.platform()) ? local : null;
+ }
+
+ /**
+ * Pulls the image and inspects what the reference names afterwards.
+ *
+ * @param platform the platform to ask the registry for, or null for the daemon's own
+ */
+ private LocalImage pull(ImageKey imageKey, String platform) {
+ String imageUri = imageKey.imageUri();
+ LOG.infov("Pulling image: {0}", imageUri);
+ AtomicReference<ManifestDigestCallback> lastAttempt = new AtomicReference<>();
+ try {
+ runWithRetry(imageUri, MAX_PULL_ATTEMPTS, INITIAL_BACKOFF_MS, () -> {
+ PullImageCmd pullImage = dockerClient.pullImageCmd(imageUri)
+ .withAuthConfig(resolveAuth(imageUri));
+ if (platform != null) {
+ pullImage.withPlatform(platform);
+ }
+ ManifestDigestCallback callback = new ManifestDigestCallback();
+ lastAttempt.set(callback);
+ pullImage.<PullImageResultCallback>exec(callback)
+ .awaitCompletion(5, TimeUnit.MINUTES);
+ });
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while pulling image: " + imageUri, e);
+ }
+ InspectImageResponse pulled = inspectLocalImage(imageUri);
+ pulledImages.add(imageKey);
+ LOG.infov("Image pulled successfully: {0}", imageUri);
+ return new LocalImage(pulled, lastAttempt.get().digest);
+ }
+
+ /**
+ * The digest of the manifest the reference resolved to, from the repository it names. A
+ * reference pinned by digest is its own answer; otherwise it is the image's repo digest for
+ * that repository, which Docker records when it pulls or pushes the image. An image can hold
+ * several digests for one repository: on the classic image store, manifests that share one
+ * image config; on the containerd store, an index and its platform manifest. The containerd
+ * store's image id is the digest of the manifest the tag names, so a digest equal to the id
+ * is the answer there. Otherwise nothing local says which one the tag names, and a launch that
+ * did not pull reports none rather than a digest the tag may no longer name.
+ */
+ static Optional<String> manifestDigest(String imageUri, InspectImageResponse image) {
+ int at = imageUri.indexOf('@');
+ if (at >= 0) {
+ return Optional.of(imageUri.substring(at + 1));
+ }
+ List<String> repoDigests = image.getRepoDigests();
+ if (repoDigests == null) {
+ return Optional.empty();
+ }
+ String repository = normalizedRepository(imageUri);
+ Set<String> digests = new LinkedHashSet<>();
+ for (String repoDigest : repoDigests) {
+ int separator = repoDigest.indexOf('@');
+ if (separator > 0 && normalizedRepository(repoDigest.substring(0, separator)).equals(repository)) {
+ digests.add(repoDigest.substring(separator + 1));
}
}
+ if (digests.size() > 1) {
+ if (digests.contains(image.getId())) {
+ return Optional.of(image.getId());
+ }
+ LOG.debugv("Image {0} holds several digests for {1}, reporting none: {2}",
+ image.getId(), imageUri, digests);
+ return Optional.empty();
+ }
+ return digests.stream().findFirst();
+ }
+
+ /** The repository a reference names, without its tag and with Docker Hub's implied prefixes. */
+ private static String normalizedRepository(String reference) {
+ String repository = reference;
+ int lastSlash = repository.lastIndexOf('/');
+ int tagSeparator = repository.lastIndexOf(':');
+ if (tagSeparator > lastSlash) {
+ repository = repository.substring(0, tagSeparator);
+ }
+ for (String prefix : DOCKER_HUB_PREFIXES) {
+ if (repository.startsWith(prefix)) {
+ repository = repository.substring(prefix.length());
+ }
+ }
+ return repository;
}
/**
@@ -220,6 +383,28 @@ public class ImageCacheService {
private record ImageKey(String imageUri, String platform) {}
+ /**
+ * The image a reference names locally.
+ *
+ * @param pulledDigest the manifest digest the registry reported when this launch pulled the
+ * image, null when the image came from the cache
+ */
+ private record LocalImage(InspectImageResponse image, String pulledDigest) {}
+
+ /** Records the manifest digest the daemon reports once a pull has resolved the reference. */
+ private static final class ManifestDigestCallback extends PullImageResultCallback {
+ private volatile String digest;
+
+ @Override
+ public void onNext(PullResponseItem item) {
+ String status = item.getStatus();
+ if (status != null && status.startsWith(DIGEST_STATUS)) {
+ digest = status.substring(DIGEST_STATUS.length()).trim();
+ }
+ super.onNext(item);
+ }
+ }
+
private AuthConfig resolveAuth(String imageUri) {
String host = extractRegistryHost(imageUri);
for (EmulatorConfig.DockerConfig.RegistryCredential cred : registryCredentials) {
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 82adf036c7..ecfe91838b 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -606,6 +606,7 @@ floci:
# Remove the ECS containers a previous run of this deployment left on the daemon before
# the service scheduler starts replacements. Scoped to this deployment's own containers.
reconcile-containers-on-startup: true
+ image-pull-behavior: default
task-role-credentials:
enabled: false
ttl-seconds: 21600
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilderTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilderTest.java
index eb1c5e4e38..2ec2f369c9 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilderTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilderTest.java
@@ -315,6 +315,15 @@ class ContainerBuilderTest {
assertEquals("ghcr.io/floci-io/mirror/floci/ami-ubuntu:24.04-arm64", spec.image());
}
+ @Test
+ void imageRegistryBaseLeavesAnImageIdAlone() {
+ TestFixture fixture = new TestFixture();
+ when(fixture.docker.imageRegistryBase()).thenReturn(Optional.of("ghcr.io/floci-io/mirror"));
+ String imageId = "sha256:" + "a".repeat(64);
+
+ assertEquals(imageId, fixture.builder.newContainer(imageId).build().image());
+ }
+
private static class TestFixture {
final EmulatorConfig config = mock(EmulatorConfig.class);
final EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsImagePullDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsImagePullDockerIntegrationTest.java
new file mode 100644
index 0000000000..cbc2253f40
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsImagePullDockerIntegrationTest.java
@@ -0,0 +1,233 @@
+package io.github.hectorvent.floci.services.ecs;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.async.ResultCallback;
+import com.github.dockerjava.api.command.CreateContainerResponse;
+import com.github.dockerjava.api.command.PullImageResultCallback;
+import com.github.dockerjava.api.exception.NotFoundException;
+import com.github.dockerjava.api.model.ExposedPort;
+import com.github.dockerjava.api.model.HostConfig;
+import com.github.dockerjava.api.model.PortBinding;
+import com.github.dockerjava.api.model.Ports;
+import com.github.dockerjava.api.model.PushResponseItem;
+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.EcsTask;
+import io.github.hectorvent.floci.services.ecs.model.NetworkMode;
+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import io.github.hectorvent.floci.testing.TestImages;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+/**
+ * The scenario from a CI rebuild: a service's image tag is moved in its registry while the Docker
+ * host still holds the old image under that tag. The next task has to run the image the registry
+ * names now, as the ECS agent's default pull behaviour does, and report that image's digest.
+ */
+@QuarkusTest
+@TestProfile(EcsImagePullDockerIntegrationTest.DockerEcsProfile.class)
+class EcsImagePullDockerIntegrationTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String MANIFEST_TYPES = String.join(",",
+ "application/vnd.oci.image.index.v1+json",
+ "application/vnd.oci.image.manifest.v1+json",
+ "application/vnd.docker.distribution.manifest.list.v2+json",
+ "application/vnd.docker.distribution.manifest.v2+json");
+
+ /**
+ * Real containers, security-group enforcement back to its shipped default of off, and the
+ * shipped pull behaviour, which the shared test config swaps for prefer-cached.
+ */
+ public static final class DockerEcsProfile implements QuarkusTestProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return Map.of("floci.services.ecs.mock", "false",
+ "floci.network.security-group-enforcement.enabled", "false",
+ "floci.services.ecs.image-pull-behavior", "default");
+ }
+ }
+
+ @Inject
+ EcsService ecsService;
+
+ @Inject
+ DockerClient dockerClient;
+
+ private String registryContainerId;
+ private final List<String> images = new ArrayList<>();
+ private final List<String> taskArns = new ArrayList<>();
+ private String clusterName;
+
+ @BeforeEach
+ void requireDocker() {
+ Assumptions.assumeTrue(isDockerAvailable(), "Docker daemon must be available for the ECS image pull test");
+ }
+
+ @AfterEach
+ void cleanUp() {
+ for (String taskArn : taskArns) {
+ ecsService.stopTask(clusterName, taskArn, "test teardown", REGION);
+ }
+ for (String image : images) {
+ try {
+ dockerClient.removeImageCmd(image).withForce(true).exec();
+ } catch (NotFoundException expected) {
+ // Removing one tag of an image can take the others with it.
+ }
+ }
+ if (registryContainerId != null) {
+ dockerClient.removeContainerCmd(registryContainerId).withForce(true).exec();
+ }
+ }
+
+ @Test
+ void aTaskRunsTheImageItsTagWasMovedToInTheRegistry() throws Exception {
+ String registry = startRegistry();
+ String repository = "floci-ecs-pull-" + UUID.randomUUID().toString().substring(0, 8);
+ String image = registry + "/" + repository + ":latest";
+ images.add(image);
+
+ pull(TestImages.BUSYBOX);
+ dockerClient.tagImageCmd(TestImages.BUSYBOX, registry + "/" + repository, "latest").exec();
+ push(registry + "/" + repository);
+ String firstDigest = registryDigest(registry, repository);
+
+ clusterName = "pull-" + UUID.randomUUID().toString().substring(0, 8);
+ ecsService.createCluster(clusterName, REGION);
+ TaskDefinition taskDef = registerTaskDefinition(image);
+
+ EcsTask first = runTask(taskDef);
+ assertEquals(firstDigest, first.getContainers().getFirst().getImageDigest());
+ String firstImageId = containerImageId(first);
+
+ // A rebuild pushed under the same tag, while the Docker host keeps the old image under it.
+ String rebuilt = commitChangedImage(image);
+ images.add(rebuilt);
+ dockerClient.tagImageCmd(rebuilt, registry + "/" + repository, "latest").exec();
+ push(registry + "/" + repository);
+ String rebuiltDigest = registryDigest(registry, repository);
+ assertNotEquals(firstDigest, rebuiltDigest, "the rebuild must have moved the tag in the registry");
+ dockerClient.tagImageCmd(firstImageId, registry + "/" + repository, "latest").exec();
+
+ EcsTask second = runTask(taskDef);
+
+ assertEquals(rebuiltDigest, second.getContainers().getFirst().getImageDigest(),
+ "the task must report the digest the registry's tag names now");
+ assertNotEquals(firstImageId, containerImageId(second),
+ "the task must run the rebuilt image, not the one the Docker host held under the tag");
+ }
+
+ private EcsTask runTask(TaskDefinition taskDef) {
+ EcsTask task = ecsService.runTask(clusterName, taskDef.getTaskDefinitionArn(), 1,
+ null, null, null, null, null, REGION).getFirst();
+ taskArns.add(task.getTaskArn());
+ EcsTask described = ecsService.describeTasks(clusterName, List.of(task.getTaskArn()), REGION).getFirst();
+ assertEquals("RUNNING", described.getLastStatus(), "the container must be up: " + described.getStoppedReason());
+ return described;
+ }
+
+ private TaskDefinition registerTaskDefinition(String image) {
+ ContainerDefinition app = new ContainerDefinition();
+ app.setName("app");
+ app.setImage(image);
+ app.setCommand(List.of("sleep", "120"));
+ return ecsService.registerTaskDefinition("pull-" + UUID.randomUUID().toString().substring(0, 8),
+ List.of(app), NetworkMode.bridge, null, null, null, null, null, REGION);
+ }
+
+ private String containerImageId(EcsTask task) {
+ return dockerClient.inspectContainerCmd(task.getContainers().getFirst().getRuntimeId()).exec().getImageId();
+ }
+
+ private String commitChangedImage(String image) {
+ CreateContainerResponse container = dockerClient.createContainerCmd(image).withCmd("true").exec();
+ try {
+ return dockerClient.commitCmd(container.getId())
+ .withLabels(Map.of("floci.test.build", UUID.randomUUID().toString()))
+ .exec();
+ } finally {
+ dockerClient.removeContainerCmd(container.getId()).withForce(true).exec();
+ }
+ }
+
+ private String startRegistry() throws InterruptedException {
+ pull(TestImages.REGISTRY);
+ ExposedPort registryPort = ExposedPort.tcp(5000);
+ registryContainerId = dockerClient.createContainerCmd(TestImages.REGISTRY)
+ .withExposedPorts(registryPort)
+ .withHostConfig(HostConfig.newHostConfig().withPortBindings(new PortBinding(
+ Ports.Binding.bindIp("127.0.0.1"), registryPort)))
+ .exec()
+ .getId();
+ dockerClient.startContainerCmd(registryContainerId).exec();
+ Ports.Binding binding = dockerClient.inspectContainerCmd(registryContainerId).exec()
+ .getNetworkSettings().getPorts().getBindings().get(registryPort)[0];
+ String registry = "localhost:" + binding.getHostPortSpec();
+ HttpClient http = HttpClient.newHttpClient();
+ for (int attempt = 0; attempt < 50; attempt++) {
+ try {
+ HttpResponse<Void> response = http.send(
+ HttpRequest.newBuilder(URI.create("http://" + registry + "/v2/")).build(),
+ HttpResponse.BodyHandlers.discarding());
+ if (response.statusCode() == 200) {
+ return registry;
+ }
+ } catch (IOException expected) {
+ // Safe to retry: the registry takes a moment to bind after the container starts.
+ }
+ Thread.sleep(100);
+ }
+ throw new IllegalStateException("registry at " + registry + " never answered");
+ }
+
+ /** The digest the registry serves for the tag, which is what a pull of it resolves to. */
+ private static String registryDigest(String registry, String repository) throws Exception {
+ HttpResponse<Void> response = HttpClient.newHttpClient().send(
+ HttpRequest.newBuilder(URI.create("http://" + registry + "/v2/" + repository + "/manifests/latest"))
+ .method("HEAD", HttpRequest.BodyPublishers.noBody())
+ .header("Accept", MANIFEST_TYPES)
+ .build(),
+ HttpResponse.BodyHandlers.discarding());
+ assertEquals(200, response.statusCode());
+ return response.headers().firstValue("Docker-Content-Digest").orElseThrow();
+ }
+
+ private void pull(String image) throws InterruptedException {
+ dockerClient.pullImageCmd(image).exec(new PullImageResultCallback()).awaitCompletion(5, TimeUnit.MINUTES);
+ }
+
+ private void push(String repository) throws InterruptedException {
+ dockerClient.pushImageCmd(repository).withTag("latest")
+ .exec(new ResultCallback.Adapter<PushResponseItem>())
+ .awaitCompletion(2, TimeUnit.MINUTES);
+ }
+
+ private boolean isDockerAvailable() {
+ try {
+ dockerClient.pingCmd().exec();
+ return true;
+ } catch (RuntimeException e) {
+ return false;
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerAwsBaselineTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerAwsBaselineTest.java
index b2d4610f6a..045a85439a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerAwsBaselineTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerAwsBaselineTest.java
@@ -14,6 +14,7 @@ import io.github.hectorvent.floci.services.ecs.model.ContainerOverride;
import io.github.hectorvent.floci.services.ecs.model.EcsTask;
import io.github.hectorvent.floci.services.ecs.model.KeyValuePair;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.ssm.SsmService;
@@ -60,8 +61,11 @@ class EcsContainerManagerAwsBaselineTest {
builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
containerBuilder = mock(ContainerBuilder.class);
when(containerBuilder.newContainer(anyString())).thenReturn(builder);
+ when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));
ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
when(lifecycleManager.createAndStart(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of()));
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEcrRewriteTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEcrRewriteTest.java
index d362436bba..0ea1654c0b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEcrRewriteTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEcrRewriteTest.java
@@ -13,6 +13,7 @@ import io.github.hectorvent.floci.services.ecs.model.Container;
import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
import io.github.hectorvent.floci.services.ecs.model.EcsTask;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.ssm.SsmService;
@@ -61,8 +62,11 @@ class EcsContainerManagerEcrRewriteTest {
ContainerBuilder.Builder builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
containerBuilder = mock(ContainerBuilder.class);
when(containerBuilder.newContainer(anyString())).thenReturn(builder);
+ when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));
lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
when(lifecycleManager.createAndStart(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of()));
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java
index 894c945330..977f58d2bf 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java
@@ -23,6 +23,7 @@ import io.github.hectorvent.floci.services.ecs.model.LogConfiguration;
import io.github.hectorvent.floci.services.ecs.model.NetworkMode;
import io.github.hectorvent.floci.services.ecs.model.PortMapping;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
@@ -71,8 +72,11 @@ class EcsContainerManagerFirelensTest {
when(builder.build()).thenReturn(new ContainerSpec("image"));
ContainerBuilder containerBuilder = mock(ContainerBuilder.class);
when(containerBuilder.newContainer(anyString())).thenReturn(builder);
+ when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));
lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("app-id", Map.of()));
when(lifecycleManager.create(any())).thenReturn("router-id");
when(lifecycleManager.startCreated(anyString(), any())).thenReturn(new ContainerInfo("router-id", Map.of()));
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerImagePullTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerImagePullTest.java
new file mode 100644
index 0000000000..cdb7445a9b
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerImagePullTest.java
@@ -0,0 +1,160 @@
+package io.github.hectorvent.floci.services.ecs.container;
+
+import com.github.dockerjava.api.exception.NotFoundException;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.config.EmulatorConfig.EcsServiceConfig.ImagePullBehavior;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;
+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
+import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;
+import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
+import io.github.hectorvent.floci.services.ecs.model.Container;
+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
+import io.github.hectorvent.floci.services.ecs.model.EcsTask;
+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
+import io.github.hectorvent.floci.services.ssm.SsmService;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.InOrder;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.RETURNS_SELF;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * A task pulls its images at launch, as the ECS agent does under {@code ECS_IMAGE_PULL_BEHAVIOR},
+ * so a tag moved in its registry is what the next task runs, and each container reports the
+ * digest of the image it runs.
+ */
+class EcsContainerManagerImagePullTest {
+
+ private static final String APP_IMAGE = "registry.example:5000/app:latest";
+ private static final String SIDECAR_IMAGE = "sidecar:latest";
+ private static final String APP_IMAGE_ID = "sha256:" + "a".repeat(64);
+ private static final String SIDECAR_IMAGE_ID = "sha256:" + "b".repeat(64);
+
+ private ContainerBuilder containerBuilder;
+ private ContainerLifecycleManager lifecycleManager;
+ private EcsContainerManager manager;
+
+ @BeforeEach
+ void setUp() {
+ ContainerBuilder.Builder builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
+ containerBuilder = mock(ContainerBuilder.class);
+ when(containerBuilder.newContainer(anyString())).thenReturn(builder);
+ when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));
+
+ lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("docker-id", Map.of()));
+
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().imagePullBehavior()).thenReturn(ImagePullBehavior.ALWAYS);
+ LaunchedContainerAwsEnv awsEnv = mock(LaunchedContainerAwsEnv.class);
+ when(awsEnv.sdkBaselineEnv(any(), any())).thenReturn(List.of());
+ EcrRegistryManager ecrRegistryManager = mock(EcrRegistryManager.class);
+ when(ecrRegistryManager.rewriteImageUri(anyString())).thenAnswer(invocation -> invocation.getArgument(0));
+
+ manager = new EcsContainerManager(containerBuilder, lifecycleManager, mock(ContainerLogStreamer.class),
+ mock(ContainerDetector.class), config, mock(RegionResolver.class), awsEnv,
+ mock(SsmService.class), mock(SecretsManagerService.class), mock(S3Service.class),
+ ecrRegistryManager, mock(HostVolumePolicy.class));
+ }
+
+ @Test
+ void everyImageIsPulledWithTheConfiguredBehaviourBeforeAnyContainerIsCreated() {
+ when(lifecycleManager.resolveImageForLaunch(APP_IMAGE, ImagePullBehavior.ALWAYS))
+ .thenReturn(new LaunchImage(APP_IMAGE_ID, "sha256:new-manifest"));
+ when(lifecycleManager.resolveImageForLaunch(SIDECAR_IMAGE, ImagePullBehavior.ALWAYS))
+ .thenReturn(new LaunchImage(SIDECAR_IMAGE_ID, null));
+ EcsTask task = task();
+
+ manager.startTask(task, taskDefinition(), List.of(), "us-east-1");
+
+ InOrder order = inOrder(lifecycleManager);
+ order.verify(lifecycleManager).resolveImageForLaunch(APP_IMAGE, ImagePullBehavior.ALWAYS);
+ order.verify(lifecycleManager).resolveImageForLaunch(SIDECAR_IMAGE, ImagePullBehavior.ALWAYS);
+ order.verify(lifecycleManager).createAndStart(any());
+ List<Container> containers = task.getContainers();
+ assertEquals("sha256:new-manifest", containers.get(0).getImageDigest());
+ assertNull(containers.get(1).getImageDigest(), "a locally built image has no manifest digest");
+ assertEquals(APP_IMAGE, containers.get(0).getImage());
+ }
+
+ @Test
+ void everyContainerIsCreatedFromTheImageIdItsPullResolvedTo() {
+ when(lifecycleManager.resolveImageForLaunch(APP_IMAGE, ImagePullBehavior.ALWAYS))
+ .thenReturn(new LaunchImage(APP_IMAGE_ID, "sha256:new-manifest"));
+ when(lifecycleManager.resolveImageForLaunch(SIDECAR_IMAGE, ImagePullBehavior.ALWAYS))
+ .thenReturn(new LaunchImage(SIDECAR_IMAGE_ID, null));
+
+ manager.startTask(task(), taskDefinition(), List.of(), "us-east-1");
+
+ verify(containerBuilder).newContainer(APP_IMAGE_ID);
+ verify(containerBuilder).newContainer(SIDECAR_IMAGE_ID);
+ verify(containerBuilder, never()).newContainer(APP_IMAGE);
+ verify(containerBuilder, never()).newContainer(SIDECAR_IMAGE);
+ }
+
+ @Test
+ void theImagePulledIsTheOneTheConfiguredRegistryBaseNames() {
+ when(containerBuilder.resolveImage(anyString()))
+ .thenAnswer(invocation -> "mirror.example/" + invocation.getArgument(0));
+ when(lifecycleManager.resolveImageForLaunch(anyString(), any()))
+ .thenReturn(new LaunchImage(APP_IMAGE_ID, null));
+
+ manager.startTask(task(), taskDefinition(), List.of(), "us-east-1");
+
+ verify(lifecycleManager).resolveImageForLaunch("mirror.example/" + APP_IMAGE, ImagePullBehavior.ALWAYS);
+ verify(lifecycleManager).resolveImageForLaunch("mirror.example/" + SIDECAR_IMAGE, ImagePullBehavior.ALWAYS);
+ }
+
+ @Test
+ void aFailedPullLeavesNoContainerCreated() {
+ when(lifecycleManager.resolveImageForLaunch(APP_IMAGE, ImagePullBehavior.ALWAYS))
+ .thenReturn(new LaunchImage(APP_IMAGE_ID, "sha256:new-manifest"));
+ when(lifecycleManager.resolveImageForLaunch(SIDECAR_IMAGE, ImagePullBehavior.ALWAYS))
+ .thenThrow(new NotFoundException("pull access denied for sidecar"));
+
+ assertThrows(NotFoundException.class,
+ () -> manager.startTask(task(), taskDefinition(), List.of(), "us-east-1"));
+
+ verify(lifecycleManager, never()).createAndStart(any());
+ }
+
+ private static TaskDefinition taskDefinition() {
+ ContainerDefinition app = new ContainerDefinition();
+ app.setName("app");
+ app.setImage(APP_IMAGE);
+ ContainerDefinition sidecar = new ContainerDefinition();
+ sidecar.setName("sidecar");
+ sidecar.setImage(SIDECAR_IMAGE);
+ TaskDefinition taskDef = new TaskDefinition();
+ taskDef.setFamily("app-service");
+ taskDef.setContainerDefinitions(List.of(app, sidecar));
+ return taskDef;
+ }
+
+ private static EcsTask task() {
+ EcsTask task = new EcsTask();
+ task.setTaskArn("arn:aws:ecs:us-east-1:000000000000:task/test-cluster/abc123");
+ return task;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerOverridesTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerOverridesTest.java
index 7fa5c040d5..37be1c99cb 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerOverridesTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerOverridesTest.java
@@ -14,6 +14,7 @@ import io.github.hectorvent.floci.services.ecs.model.ContainerOverride;
import io.github.hectorvent.floci.services.ecs.model.EcsTask;
import io.github.hectorvent.floci.services.ecs.model.KeyValuePair;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.ssm.SsmService;
@@ -59,8 +60,11 @@ class EcsContainerManagerOverridesTest {
builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
containerBuilder = mock(ContainerBuilder.class);
when(containerBuilder.newContainer(anyString())).thenReturn(builder);
+ when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));
lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
when(lifecycleManager.createAndStart(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of()));
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerPortMappingsTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerPortMappingsTest.java
index 701d2e87a2..2bd73dd3c5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerPortMappingsTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerPortMappingsTest.java
@@ -18,6 +18,7 @@ import io.github.hectorvent.floci.services.ecs.model.LogConfiguration;
import io.github.hectorvent.floci.services.ecs.model.NetworkMode;
import io.github.hectorvent.floci.services.ecs.model.PortMapping;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.ssm.SsmService;
@@ -76,8 +77,11 @@ class EcsContainerManagerPortMappingsTest {
builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
containerBuilder = mock(ContainerBuilder.class);
when(containerBuilder.newContainer(anyString())).thenReturn(builder);
+ when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));
lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
when(lifecycleManager.createAndStart(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of()));
// resolveNetworkBindings() inspects the container after launch; deep stubs
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecretsTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecretsTest.java
index 902c1a18f0..61410d85cc 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecretsTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecretsTest.java
@@ -16,6 +16,7 @@ import io.github.hectorvent.floci.services.ecs.model.EcsTask;
import io.github.hectorvent.floci.services.ecs.model.KeyValuePair;
import io.github.hectorvent.floci.services.ecs.model.Secret;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;
@@ -53,8 +54,11 @@ class EcsContainerManagerSecretsTest {
builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
containerBuilder = mock(ContainerBuilder.class);
when(containerBuilder.newContainer(anyString())).thenReturn(builder);
+ when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));
lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
when(lifecycleManager.createAndStart(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of()));
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecurityGroupTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecurityGroupTest.java
index 0085ce351e..89472740d3 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecurityGroupTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerSecurityGroupTest.java
@@ -22,6 +22,7 @@ import io.github.hectorvent.floci.services.ecs.model.NetworkConfiguration;
import io.github.hectorvent.floci.services.ecs.model.NetworkMode;
import io.github.hectorvent.floci.services.ecs.model.PortMapping;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.ssm.SsmService;
@@ -66,8 +67,11 @@ class EcsContainerManagerSecurityGroupTest {
builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
ContainerBuilder containerBuilder = mock(ContainerBuilder.class);
when(containerBuilder.newContainer(anyString())).thenReturn(builder);
+ when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));
lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("docker-id", Map.of()));
dockerClient = mock(DockerClient.class, RETURNS_DEEP_STUBS);
when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerTaskRoleCredentialsTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerTaskRoleCredentialsTest.java
index e089c61e5f..ed931e5422 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerTaskRoleCredentialsTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerTaskRoleCredentialsTest.java
@@ -14,6 +14,7 @@ import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
import io.github.hectorvent.floci.services.ecs.model.EcsTask;
import io.github.hectorvent.floci.services.ecs.model.KeyValuePair;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.ssm.SsmService;
@@ -68,9 +69,12 @@ class EcsContainerManagerTaskRoleCredentialsTest {
builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
containerBuilder = mock(ContainerBuilder.class);
when(containerBuilder.newContainer(anyString())).thenReturn(builder);
+ when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));
when(containerBuilder.resolveDockerNetwork(any())).thenReturn(Optional.of(NETWORK));
ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("docker-id", Map.of()));
config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
@@ -184,6 +188,8 @@ class EcsContainerManagerTaskRoleCredentialsTest {
@Test
void revokesCredentialsEvenWhenAContainerCouldNotBeRemoved() {
ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
DockerClient dockerClient = mock(DockerClient.class, RETURNS_DEEP_STUBS);
when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
// Docker refuses the removal, so the task's containers never all come off.
@@ -227,6 +233,8 @@ class EcsContainerManagerTaskRoleCredentialsTest {
@Test
void keepsAddressesReservedWhenAContainerCouldNotBeRemoved() {
ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
DockerClient dockerClient = mock(DockerClient.class, RETURNS_DEEP_STUBS);
when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
when(dockerClient.removeContainerCmd("docker-id")).thenThrow(new IllegalStateException("in use"));
@@ -249,6 +257,8 @@ class EcsContainerManagerTaskRoleCredentialsTest {
@Test
void keepsAddressesReservedWhenCleanupCannotRemoveAContainer() {
ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("docker-id", Map.of()));
// The reconciler's cleanup path is the other way a task's containers come off, and Docker
// can refuse a removal here just as it can on the stop path.
@@ -274,6 +284,8 @@ class EcsContainerManagerTaskRoleCredentialsTest {
@Test
void keepsAddressesReservedWhenAFailedLaunchCannotRemoveAContainer() {
ContainerLifecycleManager failing = mock(ContainerLifecycleManager.class);
+ when(failing.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
// The first container starts and the second does not, so the launch unwinds with one
// container already up and holding an address.
when(failing.createAndStart(any()))
@@ -349,6 +361,8 @@ class EcsContainerManagerTaskRoleCredentialsTest {
@Test
void revokesCredentialsWhenTheLaunchFailsPartWayThrough() {
ContainerLifecycleManager failing = mock(ContainerLifecycleManager.class);
+ when(failing.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
when(failing.createAndStart(any())).thenThrow(new IllegalStateException("no daemon"));
EcsContainerManager failingManager = new EcsContainerManager(containerBuilder, failing,
mock(ContainerLogStreamer.class), mock(ContainerDetector.class), config,
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java
index 706369af09..e98ce3b292 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java
@@ -23,6 +23,7 @@ import io.github.hectorvent.floci.services.ecs.model.FirelensConfiguration;
import io.github.hectorvent.floci.services.ecs.model.LogConfiguration;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
import io.github.hectorvent.floci.services.ecs.model.VolumeFrom;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.ssm.SsmService;
@@ -61,6 +62,7 @@ class EcsContainerManagerVolumesFromTest {
@BeforeEach
void setUp() {
containerBuilder = mock(ContainerBuilder.class);
+ when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));
sourceBuilder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
appBuilder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
when(containerBuilder.newContainer("sidecar:latest")).thenReturn(sourceBuilder);
@@ -69,6 +71,8 @@ class EcsContainerManagerVolumesFromTest {
when(appBuilder.build()).thenReturn(mock(ContainerSpec.class));
lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
when(lifecycleManager.createAndStart(any()))
.thenReturn(new ContainerInfo("source-id", Map.of()))
.thenReturn(new ContainerInfo("app-id", Map.of()));
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java
index 0b2f5a1366..8f7df22897 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesTest.java
@@ -16,6 +16,7 @@ import io.github.hectorvent.floci.services.ecs.model.EfsVolumeConfiguration;
import io.github.hectorvent.floci.services.ecs.model.MountPoint;
import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;
import io.github.hectorvent.floci.services.ecs.model.Volume;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.ssm.SsmService;
@@ -65,8 +66,11 @@ class EcsContainerManagerVolumesTest {
builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
containerBuilder = mock(ContainerBuilder.class);
when(containerBuilder.newContainer(anyString())).thenReturn(builder);
+ when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));
lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.resolveImageForLaunch(any(), any()))
+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
when(lifecycleManager.createAndStart(any()))
.thenReturn(new ContainerInfo("docker-id", Map.of()));
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java
index 54ebdc3be1..8ef6fbd92e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheServiceTest.java
@@ -12,10 +12,16 @@ import com.github.dockerjava.api.exception.InternalServerErrorException;
import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.exception.UnauthorizedException;
import com.github.dockerjava.api.model.Info;
+import com.github.dockerjava.api.model.PullResponseItem;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.config.EmulatorConfig.EcsServiceConfig.ImagePullBehavior;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;
import org.junit.jupiter.api.Test;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -24,6 +30,7 @@ import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -34,6 +41,9 @@ import static org.mockito.Mockito.when;
class ImageCacheServiceTest {
private static final String IMAGE = "public.ecr.aws/docker/library/alpine:latest";
+ private static final String REGISTRY_REPO = "registry.example:5000/app";
+ private static final String REGISTRY_IMAGE = REGISTRY_REPO + ":latest";
+ private static final String LOCAL_IMAGE = "app-local:latest";
@Test
void pullsImageWhenInspectionReportsNotFound() throws Exception {
@@ -383,6 +393,290 @@ class ImageCacheServiceTest {
assertEquals(1, calls.get());
}
+ @Test
+ void defaultPullBehaviourRunsTheImageAMovedTagNamesNow() {
+ FakeDaemon daemon = new FakeDaemon();
+ daemon.registry.put(REGISTRY_IMAGE, image("sha256:old", REGISTRY_REPO + "@sha256:old-manifest"));
+ ImageCacheService service = newService(daemon.client);
+ assertEquals("sha256:old", service.ensureImageExists(REGISTRY_IMAGE));
+
+ daemon.registry.put(REGISTRY_IMAGE, image("sha256:new", REGISTRY_REPO + "@sha256:new-manifest"));
+
+ assertEquals(new LaunchImage("sha256:new", "sha256:new-manifest"),
+ service.resolveForLaunch(REGISTRY_IMAGE, ImagePullBehavior.DEFAULT));
+ assertEquals("sha256:new", service.ensureImageExists(REGISTRY_IMAGE));
+ assertEquals(2, daemon.pulls.get());
+ }
+
+ @Test
+ void defaultPullBehaviourRunsTheCachedImageWhenThePullFails() {
+ FakeDaemon daemon = new FakeDaemon();
+ daemon.store(LOCAL_IMAGE, image("sha256:built"));
+ ImageCacheService service = newService(daemon.client);
+
+ assertEquals(new LaunchImage("sha256:built", null),
+ service.resolveForLaunch(LOCAL_IMAGE, ImagePullBehavior.DEFAULT));
+ assertEquals("sha256:built", service.ensureImageExists(LOCAL_IMAGE));
+ assertEquals(1, daemon.pulls.get());
+ }
+
+ @Test
+ void defaultPullBehaviourFailsWhenThePullFailsAndNothingIsCached() {
+ FakeDaemon daemon = new FakeDaemon();
+
+ assertThrows(NotFoundException.class,
+ () -> newService(daemon.client).resolveForLaunch(LOCAL_IMAGE, ImagePullBehavior.DEFAULT));
+ }
+
+ @Test
+ void alwaysPullBehaviourFailsWhenThePullFailsEvenWithACachedImage() {
+ FakeDaemon daemon = new FakeDaemon();
+ daemon.store(LOCAL_IMAGE, image("sha256:built"));
+
+ assertThrows(NotFoundException.class,
+ () -> newService(daemon.client).resolveForLaunch(LOCAL_IMAGE, ImagePullBehavior.ALWAYS));
+ }
+
+ @Test
+ void preferCachedPullBehaviourFollowsTheLocalTagWithoutPulling() {
+ FakeDaemon daemon = new FakeDaemon();
+ daemon.store(REGISTRY_IMAGE, image("sha256:old", REGISTRY_REPO + "@sha256:old-manifest"));
+ ImageCacheService service = newService(daemon.client);
+ assertEquals("sha256:old", service.ensureImageExists(REGISTRY_IMAGE));
+
+ daemon.store(REGISTRY_IMAGE, image("sha256:new", REGISTRY_REPO + "@sha256:new-manifest"));
+
+ assertEquals(new LaunchImage("sha256:new", "sha256:new-manifest"),
+ service.resolveForLaunch(REGISTRY_IMAGE, ImagePullBehavior.PREFER_CACHED));
+ assertEquals("sha256:new", service.ensureImageExists(REGISTRY_IMAGE));
+ assertEquals(0, daemon.pulls.get());
+ }
+
+ @Test
+ void preferCachedPullBehaviourPullsWhenNothingIsCached() {
+ FakeDaemon daemon = new FakeDaemon();
+ daemon.registry.put(REGISTRY_IMAGE, image("sha256:new", REGISTRY_REPO + "@sha256:new-manifest"));
+
+ assertEquals(new LaunchImage("sha256:new", "sha256:new-manifest"),
+ newService(daemon.client).resolveForLaunch(REGISTRY_IMAGE, ImagePullBehavior.PREFER_CACHED));
+ assertEquals(1, daemon.pulls.get());
+ }
+
+ @Test
+ void oncePullBehaviourPullsOnlyTheFirstLaunch() {
+ FakeDaemon daemon = new FakeDaemon();
+ daemon.store(REGISTRY_IMAGE, image("sha256:stale"));
+ daemon.registry.put(REGISTRY_IMAGE, image("sha256:old", REGISTRY_REPO + "@sha256:old-manifest"));
+ ImageCacheService service = newService(daemon.client);
+
+ service.resolveForLaunch(REGISTRY_IMAGE, ImagePullBehavior.ONCE);
+ daemon.registry.put(REGISTRY_IMAGE, image("sha256:new", REGISTRY_REPO + "@sha256:new-manifest"));
+
+ assertEquals(new LaunchImage("sha256:old", "sha256:old-manifest"),
+ service.resolveForLaunch(REGISTRY_IMAGE, ImagePullBehavior.ONCE));
+ assertEquals("sha256:old", service.ensureImageExists(REGISTRY_IMAGE));
+ assertEquals(1, daemon.pulls.get());
+ }
+
+ @Test
+ void aLaunchKeepsTheImageItResolvedWhenAnOverlappingLaunchMovesTheTag() {
+ String oldId = "sha256:" + "a".repeat(64);
+ String newId = "sha256:" + "b".repeat(64);
+ FakeDaemon daemon = new FakeDaemon();
+ daemon.registry.put(REGISTRY_IMAGE, image(oldId, REGISTRY_REPO + "@sha256:old-manifest"));
+ ImageCacheService service = newService(daemon.client);
+
+ LaunchImage first = service.resolveForLaunch(REGISTRY_IMAGE, ImagePullBehavior.DEFAULT);
+ daemon.registry.put(REGISTRY_IMAGE, image(newId, REGISTRY_REPO + "@sha256:new-manifest"));
+ LaunchImage second = service.resolveForLaunch(REGISTRY_IMAGE, ImagePullBehavior.DEFAULT);
+
+ assertEquals(new LaunchImage(oldId, "sha256:old-manifest"), first);
+ assertEquals(new LaunchImage(newId, "sha256:new-manifest"), second);
+ assertEquals(oldId, service.ensureImageExists(first.imageId()),
+ "the first launch's container is created from the image its own pull resolved to");
+ assertEquals(newId, service.ensureImageExists(second.imageId()));
+ assertEquals(2, daemon.pulls.get());
+ }
+
+ @Test
+ void anImageIdThatIsGoneFailsWithoutAPull() {
+ FakeDaemon daemon = new FakeDaemon();
+
+ assertThrows(DockerClientException.class,
+ () -> newService(daemon.client).ensureImageExists("sha256:" + "c".repeat(64)));
+ assertEquals(0, daemon.pulls.get());
+ }
+
+ @Test
+ void anImageIdLaunchesWithoutAPullUnderEveryBehaviour() {
+ String imageId = "sha256:" + "d".repeat(64);
+ FakeDaemon daemon = new FakeDaemon();
+ daemon.store(LOCAL_IMAGE, image(imageId));
+ ImageCacheService service = newService(daemon.client);
+
+ for (ImagePullBehavior behavior : ImagePullBehavior.values()) {
+ assertEquals(new LaunchImage(imageId, null), service.resolveForLaunch(imageId, behavior));
+ }
+ assertEquals(0, daemon.pulls.get());
+ }
+
+ @Test
+ void anImageIdThatIsGoneFailsALaunchWithoutAPull() {
+ FakeDaemon daemon = new FakeDaemon();
+
+ assertThrows(DockerClientException.class, () -> newService(daemon.client)
+ .resolveForLaunch("sha256:" + "e".repeat(64), ImagePullBehavior.ALWAYS));
+ assertEquals(0, daemon.pulls.get());
+ }
+
+ @Test
+ void aLaunchReportsTheDigestItsPullResolvedTheTagTo() {
+ FakeDaemon daemon = new FakeDaemon();
+ daemon.registry.put(REGISTRY_IMAGE, image("sha256:config",
+ REGISTRY_REPO + "@sha256:platform-manifest", REGISTRY_REPO + "@sha256:index"));
+ daemon.registryDigests.put(REGISTRY_IMAGE, "sha256:index");
+
+ assertEquals(new LaunchImage("sha256:config", "sha256:index"),
+ newService(daemon.client).resolveForLaunch(REGISTRY_IMAGE, ImagePullBehavior.DEFAULT));
+ }
+
+ @Test
+ void aCachedLaunchReportsNoDigestWhenAnOutsidePullMovedTheTagToAnotherManifestOfTheSameImage() {
+ FakeDaemon daemon = new FakeDaemon();
+ daemon.registry.put(REGISTRY_IMAGE, image("sha256:config", REGISTRY_REPO + "@sha256:first-manifest"));
+ daemon.registryDigests.put(REGISTRY_IMAGE, "sha256:first-manifest");
+ ImageCacheService service = newService(daemon.client);
+ service.resolveForLaunch(REGISTRY_IMAGE, ImagePullBehavior.DEFAULT);
+
+ // A docker pull outside Floci moves the tag to a second manifest of the same image config,
+ // which the classic image store keeps under the same image id.
+ daemon.store(REGISTRY_IMAGE, image("sha256:config",
+ REGISTRY_REPO + "@sha256:first-manifest", REGISTRY_REPO + "@sha256:second-manifest"));
+
+ assertEquals(new LaunchImage("sha256:config", null),
+ service.resolveForLaunch(REGISTRY_IMAGE, ImagePullBehavior.PREFER_CACHED));
+ }
+
+ @Test
+ void aCachedLaunchReportsTheDigestTheContainerdStoreNamesTheImageBy() {
+ FakeDaemon daemon = new FakeDaemon();
+ daemon.store(REGISTRY_IMAGE, image("sha256:index",
+ REGISTRY_REPO + "@sha256:platform-manifest", REGISTRY_REPO + "@sha256:index"));
+
+ assertEquals(new LaunchImage("sha256:index", "sha256:index"),
+ newService(daemon.client).resolveForLaunch(REGISTRY_IMAGE, ImagePullBehavior.PREFER_CACHED));
+ }
+
+ @Test
+ void aCachedLaunchDoesNotReportTheDigestOfAnImageTheTagNoLongerNames() {
+ FakeDaemon daemon = new FakeDaemon();
+ daemon.registry.put(REGISTRY_IMAGE, image("sha256:old", REGISTRY_REPO + "@sha256:old-manifest"));
+ daemon.registryDigests.put(REGISTRY_IMAGE, "sha256:old-manifest");
+ ImageCacheService service = newService(daemon.client);
+ service.resolveForLaunch(REGISTRY_IMAGE, ImagePullBehavior.DEFAULT);
+
+ daemon.store(REGISTRY_IMAGE, image("sha256:rebuilt"));
+
+ assertEquals(new LaunchImage("sha256:rebuilt", null),
+ service.resolveForLaunch(REGISTRY_IMAGE, ImagePullBehavior.PREFER_CACHED));
+ }
+
+ @Test
+ void manifestDigestIsEmptyWhenTheRepositoryHasSeveralDigests() {
+ InspectImageResponse image = image("sha256:id",
+ REGISTRY_REPO + "@sha256:platform-manifest", REGISTRY_REPO + "@sha256:index");
+
+ assertEquals(Optional.empty(), ImageCacheService.manifestDigest(REGISTRY_IMAGE, image));
+ }
+
+ @Test
+ void manifestDigestIsTheRepoDigestOfTheRepositoryTheReferenceNames() {
+ InspectImageResponse pulled = image("sha256:id",
+ "mirror.example:5000/app@sha256:mirror-manifest", REGISTRY_REPO + "@sha256:manifest");
+
+ assertEquals(Optional.of("sha256:manifest"), ImageCacheService.manifestDigest(REGISTRY_IMAGE, pulled));
+ }
+
+ @Test
+ void manifestDigestMatchesDockerHubRepositoriesWrittenInFull() {
+ InspectImageResponse pulled = image("sha256:id", "alpine@sha256:manifest");
+
+ assertEquals(Optional.of("sha256:manifest"),
+ ImageCacheService.manifestDigest("docker.io/library/alpine:3.20", pulled));
+ }
+
+ @Test
+ void manifestDigestOfADigestReferenceIsThatDigest() {
+ assertEquals(Optional.of("sha256:pinned"),
+ ImageCacheService.manifestDigest(REGISTRY_REPO + "@sha256:pinned", image("sha256:id")));
+ }
+
+ @Test
+ void manifestDigestIsEmptyForALocallyBuiltImage() {
+ assertEquals(Optional.empty(), ImageCacheService.manifestDigest(LOCAL_IMAGE, image("sha256:built")));
+ }
+
+ private static InspectImageResponse image(String id, String... repoDigests) {
+ return new InspectImageResponse()
+ .withId(id)
+ .withOs("linux")
+ .withArch("amd64")
+ .withRepoDigests(List.of(repoDigests));
+ }
+
+ /**
+ * A daemon whose local images and registry are plain maps, so a test can move a tag in either
+ * place and watch which image the next launch resolves to.
+ */
+ private static final class FakeDaemon {
+ final DockerClient client = mock(DockerClient.class);
+ final Map<String, InspectImageResponse> local = new HashMap<>();
+ final Map<String, InspectImageResponse> registry = new HashMap<>();
+ final Map<String, String> registryDigests = new HashMap<>();
+ final AtomicInteger pulls = new AtomicInteger();
+
+ FakeDaemon() {
+ when(client.inspectImageCmd(anyString())).thenAnswer(invocation -> {
+ String reference = invocation.getArgument(0);
+ InspectImageCmd inspect = mock(InspectImageCmd.class);
+ when(inspect.exec()).thenAnswer(ignored -> {
+ InspectImageResponse image = local.get(reference);
+ if (image == null) {
+ throw new NotFoundException("No such image: " + reference);
+ }
+ return image;
+ });
+ return inspect;
+ });
+ when(client.pullImageCmd(anyString())).thenAnswer(invocation -> {
+ String reference = invocation.getArgument(0);
+ PullImageCmd pull = mock(PullImageCmd.class);
+ when(pull.withAuthConfig(any())).thenReturn(pull);
+ when(pull.exec(any(PullImageResultCallback.class))).thenAnswer(execution -> {
+ pulls.incrementAndGet();
+ InspectImageResponse image = registry.get(reference);
+ if (image == null) {
+ throw new NotFoundException("manifest unknown: " + reference);
+ }
+ store(reference, image);
+ String digest = registryDigests.get(reference);
+ if (digest != null) {
+ PullResponseItem status = mock(PullResponseItem.class);
+ when(status.getStatus()).thenReturn("Digest: " + digest);
+ execution.<PullImageResultCallback>getArgument(0).onNext(status);
+ }
+ return mock(PullImageResultCallback.class);
+ });
+ return pull;
+ });
+ }
+
+ void store(String tag, InspectImageResponse image) {
+ local.put(tag, image);
+ local.put(image.getId(), image);
+ }
+ }
+
private static ImageCacheService newService(DockerClient dockerClient) {
InfoCmd infoCmd = mock(InfoCmd.class);
when(dockerClient.infoCmd()).thenReturn(infoCmd);
diff --git a/src/test/java/io/github/hectorvent/floci/testing/TestImages.java b/src/test/java/io/github/hectorvent/floci/testing/TestImages.java
index 973786e373..55a2d78a95 100644
--- a/src/test/java/io/github/hectorvent/floci/testing/TestImages.java
+++ b/src/test/java/io/github/hectorvent/floci/testing/TestImages.java
@@ -30,6 +30,9 @@ public final class TestImages {
/** The same image AWS publishes at {@code public.ecr.aws/aws-observability/aws-for-fluent-bit:3}. */
public static final String FLUENT_BIT = "amazon/aws-for-fluent-bit:3";
+ /** The tag {@code floci.services.ecr.registry-image} ships with. */
+ public static final String REGISTRY = "registry:2";
+
private TestImages() {
}
}
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index 1e66f380d5..5280659310 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -317,6 +317,7 @@ floci:
mock: true
host-volume-roots: []
allow-unsafe-host-volumes: false
+ image-pull-behavior: prefer-cached
appconfig:
enabled: true
appconfigdata: