Light, fluffy, and always free - The AWS Local Emulator alternative
ci: key the ECR Public prefetch on the classes that start each image (#4600)
Kenshin Okinaka903efdf58eb6
diff --git a/.github/ci/prefetch-images.sh b/.github/ci/prefetch-images.shindex af76a179fa..f6c25a7b8b 100755--- a/.github/ci/prefetch-images.sh+++ b/.github/ci/prefetch-images.sh@@ -2,25 +2,95 @@# Best-effort background prefetch of the Docker images this shard's tests use.# Never fails the build; a pull that loses the race just means the test pulls# it itself, exactly as before. Pulls only images implied by shard.txt so four-# shards do not each pull all images (Docker Hub rate-limit exposure).+# shards do not each pull all images (registry rate-limit exposure).+# ECR Public counts anonymous transfer per runner IP, so an image is keyed on the classes that+# start it whenever its package is spread over every shard (Lambda, ECS), and a key that names a+# class is checked against src/test/java so a rename cannot silently turn the prefetch off. The+# Docker library images the tests start come from Docker Hub, which GitHub-hosted runners pull+# without a limit; TestImages says why they are not taken through the ECR Public mirror.+# Each pull writes one line to $PREFETCH_LOG (prefetch-images.log next to shard.txt by default):+# "prefetched <image>" or "prefetch failed <image>: <last line of docker's output>", timestamped,+# so a test that pulled the image itself can be told apart from a prefetch that failed or was+# still downloading. The lines go to a file because the pulls outlive the workflow step that+# started them and the step's stdout is closed by then.# Usage: prefetch-images.sh <shard.txt>set -uSHARD_FILE="${1:-}"+PREFETCH_LOG="${PREFETCH_LOG:-$(dirname "${SHARD_FILE:-.}")/prefetch-images.log}"command -v docker >/dev/null 2>&1 || exit 0[ -r "$SHARD_FILE" ] || { echo "prefetch-images: ${SHARD_FILE:-<none>} not readable, nothing prefetched" >&2; exit 0; }+: > "$PREFETCH_LOG"-pull() { docker pull -q "$1" >/dev/null 2>&1 & }+pull_and_log() {+ if output="$(docker pull -q "$1" 2>&1)"; then+ echo "$(date -u +%H:%M:%S) prefetched $1" >> "$PREFETCH_LOG"+ else+ echo "$(date -u +%H:%M:%S) prefetch failed $1: $(printf '%s\n' "$output" | tail -n 1)" >> "$PREFETCH_LOG"+ fi+}+# Docker Hub and other registries: each pull in its own background job, as before. An image two+# entries name (busybox:stable is both the EFS init image and the tests' shell) is pulled once.+REQUESTED=""+pull() {+ case " $REQUESTED " in *" $1 "*) return 0;; esac+ REQUESTED="$REQUESTED $1"+ pull_and_log "$1" &+}+# ECR Public counts unauthenticated pulls per second as well as bytes, so its images are queued+# and pulled one after another, a second apart, in a single background job instead of all at once.+ECR_PUBLIC_IMAGES=""+ecr() { ECR_PUBLIC_IMAGES="$ECR_PUBLIC_IMAGES $1"; }+# True when the shard runs any of the named classes. A name with no source file is a stale key,+# recorded in the log so the next reader can see why an image stopped being prefetched.+starts() {+ for class in "$@"; do+ [ -n "$(find src/test/java -name "$class.java" -print -quit)" ] \+ || echo "$(date -u +%H:%M:%S) stale prefetch key: no test class named $class" >> "$PREFETCH_LOG"+ done+ grep -qE "$(IFS='|'; echo "$*")" "$SHARD_FILE"+}+# The image a test constant pins, or empty (and a log line) when the constant cannot be read.+test_image() {+ image="$(grep -oE "$1 *= *\"[^\"]+\"" src/test/java/io/github/hectorvent/floci/testing/TestImages.java | grep -oE '"[^"]+"' | tr -d '"')"+ [ -n "$image" ] || echo "$(date -u +%H:%M:%S) prefetch skipped: TestImages.$1 not readable" >> "$PREFETCH_LOG"+ echo "$image"+}++# Every Docker-backed test that needs a shell uses the one busybox TestImages pins, so one pull+# serves all of them. It is also the base of the image+# EcsContainerManagerVolumesFromDockerIntegrationTest builds, and a build resolves its base against+# the registry rather than through ImageCacheService, so it is prefetched for the cache hit rather+# than for the second it saves. ContainerPlatformDockerIntegrationTest is not listed: it pulls the+# other architecture, which a prefetch of the host image cannot serve.+BUSYBOX_IMAGE="$(test_image BUSYBOX)"+starts ContainerHostNetworkDockerIntegrationTest EcsServiceDiscoveryDockerIntegrationTest \+ EcsContainerManagerEfsIsolationDockerIntegrationTest EcsContainerManagerFirelensDockerIntegrationTest \+ EcsContainerManagerStatsDockerIntegrationTest EcsContainerManagerVolumesFromDockerIntegrationTest \+ EcsExecChannelDockerIntegrationTest BatchDockerRunnerDockerIntegrationTest SageMakerDockerIntegrationTest \+ && [ -n "$BUSYBOX_IMAGE" ] && pull "$BUSYBOX_IMAGE"+PYTHON_ALPINE_IMAGE="$(test_image PYTHON_ALPINE)"+starts ContainerCaBundleDockerIntegrationTest EcsCredentialsProxyDockerIntegrationTest SageMakerDockerIntegrationTest \+ && [ -n "$PYTHON_ALPINE_IMAGE" ] && pull "$PYTHON_ALPINE_IMAGE"+# ECR Public images, in queue order: the Lambda runtimes by size, then the Firelens router.+# Lambda runtimes, measured from a full four-shard run by pairing each ImageCacheService+# "Pulling image" / "Image already present locally" line with the class that logged it.+# nodejs:20 is the runtime the API Gateway, ELBv2 and Lambda integration tests resolve and every+# shard reaches it; the two Python runtimes each have one Docker-backed class; nodejs:18 was+# pulled by every shard and started by none. A class missed here pulls inline, as before.+grep -qE '/apigateway/|/apigatewayv2/|/lambda/' "$SHARD_FILE" && ecr public.ecr.aws/lambda/nodejs:20+starts SwfLambdaIntegrationTest && ecr public.ecr.aws/lambda/python:3.12+starts ElbV2LambdaTargetDataPlaneIntegrationTest && ecr public.ecr.aws/lambda/python:3.14+# The Firelens test names the log router image itself rather than taking it from config.+starts EcsContainerManagerFirelensDockerIntegrationTest && ecr public.ecr.aws/aws-observability/aws-for-fluent-bit:3# package token in shard.txt -> images its tests are known to launch# (measured inline pulls; update alongside image-catalog changes)-grep -q '/lambda/' "$SHARD_FILE" && { pull public.ecr.aws/lambda/python:3.14; pull public.ecr.aws/lambda/nodejs:18; pull public.ecr.aws/lambda/nodejs:20; pull public.ecr.aws/lambda/python:3.12; }grep -q '/docdb/' "$SHARD_FILE" && pull mongo:7.0grep -q '/neptune/' "$SHARD_FILE" && { pull neo4j:5-community; pull tinkerpop/gremlin-server:3.7.3; }grep -q '/elasticache/' "$SHARD_FILE" && { pull valkey/valkey:8; pull memcached:1.6; }grep -q '/memorydb/' "$SHARD_FILE" && pull valkey/valkey:8grep -q '/ecr/' "$SHARD_FILE" && pull registry:2grep -q '/ec2/' "$SHARD_FILE" && { pull busybox:stable; pull alpine:latest; }-grep -q '/common/docker/' "$SHARD_FILE" && pull public.ecr.aws/docker/library/python:3.12-alpine# The Cedar sidecar pin lives in application.yml; read it rather than duplicate it.CEDAR_IMAGE="$(grep -oE 'cedar-image: *"[^"]+"' src/main/resources/application.yml | grep -oE '"[^"]+"' | tr -d '"')"grep -q '/verifiedpermissions/' "$SHARD_FILE" && [ -n "$CEDAR_IMAGE" ] && pull "$CEDAR_IMAGE"@@ -48,10 +118,7 @@ INFLUX_IMAGE="$(grep -oE 'default-image: *"influxdb:[^"]+"' src/main/resources/agrep -q '/timestreaminfluxdb/' "$SHARD_FILE" && [ -n "$INFLUX_IMAGE" ] && pull "$INFLUX_IMAGE"K3S_IMAGE="$(grep -oE 'default-image: *"rancher/k3s:[^"]+"' src/main/resources/application.yml | grep -oE '"[^"]+"' | tr -d '"')"grep -q '/eks/' "$SHARD_FILE" && [ -n "$K3S_IMAGE" ] && pull "$K3S_IMAGE"-# The Firelens test names the log router image itself rather than taking it from config.-# busybox is the base of the image EcsContainerManagerVolumesFromDockerIntegrationTest builds, and-# a build resolves its base against the registry rather than through ImageCacheService. That pull-# has hit "toomanyrequests: Rate exceeded" and failed the shard three times (PRs 4075, 4142, 4164),-# so it is prefetched for the cache hit rather than for the second it saves.-grep -q '/ecs/container/' "$SHARD_FILE" && { pull public.ecr.aws/aws-observability/aws-for-fluent-bit:3; pull public.ecr.aws/docker/library/busybox:latest; }+if [ -n "$ECR_PUBLIC_IMAGES" ]; then+ { for image in $ECR_PUBLIC_IMAGES; do pull_and_log "$image"; sleep 1; done; } &+fiexit 0diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 35de0c30a7..f376fc7d25 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -118,6 +118,15 @@ jobs:-Dsurefire.includesFile="$GITHUB_WORKSPACE/shard.txt"-Dsurefire.failIfNoTests=true+ # The prefetch runs in the background past its own step, so its per-image+ # results are only complete once the tests are done. A pull still running+ # here (after an early exit) has no result line yet, so list it.+ - name: Image prefetch results+ if: always()+ run: |+ cat prefetch-images.log 2>/dev/null || echo "no prefetch log"+ pgrep -af 'docker pull' | sed 's/^/still running: /' || true+# Save from main only, and from shard 1 only. Actions caches are ref-scoped:# a pull_request run's write lands in refs/pull/N/merge where no other PR can# read it, so every open PR was storing a ~201 MB byte-identical duplicate of@@ -144,6 +153,7 @@ jobs:path: |shard.txtall-classes.txt+ prefetch-images.logtarget/surefire-reports/if-no-files-found: errordiff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CodeBuildTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CodeBuildTest.javaindex 6b57569690..e642a8d5ee 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CodeBuildTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CodeBuildTest.java@@ -46,7 +46,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;class CodeBuildTest {private static final String CODEBUILD_TEST_IMAGE =- "public.ecr.aws/docker/library/busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d";+ "busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d";static CodeBuildClient codebuild;static String reportGroupArn;diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SageMakerTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SageMakerTest.javaindex c1ef64bbc9..344dc3825f 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SageMakerTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/SageMakerTest.java@@ -23,7 +23,7 @@ import static org.assertj.core.api.Assertions.assertThat;@TestMethodOrder(MethodOrderer.OrderAnnotation.class)class SageMakerTest {- private static final String TRAINING_IMAGE = "public.ecr.aws/docker/library/busybox:stable";+ private static final String TRAINING_IMAGE = "busybox:stable";private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/sdk-test-role";static SageMakerClient sagemaker;diff --git a/compatibility-tests/sdk-test-python/tests/test_sagemaker.py b/compatibility-tests/sdk-test-python/tests/test_sagemaker.pyindex 2c3cc99965..a9e8b03147 100644--- a/compatibility-tests/sdk-test-python/tests/test_sagemaker.py+++ b/compatibility-tests/sdk-test-python/tests/test_sagemaker.py@@ -31,7 +31,7 @@ def test_sagemaker_control_plane_and_training():cfg = f"compat-sm-cfg-{suffix}"bucket = f"compat-sm-{suffix}"- sm.create_model(ModelName=model, PrimaryContainer={"Image": "public.ecr.aws/docker/library/busybox:stable"}, ExecutionRoleArn="arn:aws:iam::000000000000:role/r")+ sm.create_model(ModelName=model, PrimaryContainer={"Image": "busybox:stable"}, ExecutionRoleArn="arn:aws:iam::000000000000:role/r")assert sm.describe_model(ModelName=model)["ModelName"] == modelsm.create_endpoint_config(EndpointConfigName=cfg, ProductionVariants=[{"VariantName": "AllTraffic", "ModelName": model, "InitialInstanceCount": 1, "InstanceType": "ml.t2.medium"}])assert any(c["EndpointConfigName"] == cfg for c in sm.list_endpoint_configs()["EndpointConfigs"])@@ -43,7 +43,7 @@ def test_sagemaker_control_plane_and_training():TrainingJobName=job,RoleArn="arn:aws:iam::000000000000:role/r",AlgorithmSpecification={- "TrainingImage": "public.ecr.aws/docker/library/busybox:stable",+ "TrainingImage": "busybox:stable","TrainingInputMode": "File","ContainerEntrypoint": ["/bin/sh", "-c"],"ContainerArguments": ["mkdir -p /opt/ml/model && echo ok > /opt/ml/model/model.txt"],diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerCaBundleDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerCaBundleDockerIntegrationTest.javaindex d772072145..252e7f8d42 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerCaBundleDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerCaBundleDockerIntegrationTest.java@@ -7,6 +7,7 @@ import com.github.dockerjava.core.command.WaitContainerResultCallback;import io.github.hectorvent.floci.config.ContainerCaBundle;import io.github.hectorvent.floci.config.FlociCertificateAuthority;import io.github.hectorvent.floci.services.acm.model.KeyAlgorithm;+import io.github.hectorvent.floci.testing.TestImages;import io.quarkus.test.junit.QuarkusTest;import io.quarkus.test.junit.QuarkusTestProfile;import io.quarkus.test.junit.TestProfile;@@ -38,7 +39,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;@TestProfile(ContainerCaBundleDockerIntegrationTest.Profile.class)class ContainerCaBundleDockerIntegrationTest {- private static final String IMAGE = "public.ecr.aws/docker/library/python:3.12-alpine";+ private static final String IMAGE = TestImages.PYTHON_ALPINE;@InjectDockerClient dockerClient;diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerHostNetworkDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerHostNetworkDockerIntegrationTest.javaindex b170413612..7c6a2d807f 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerHostNetworkDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerHostNetworkDockerIntegrationTest.java@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.core.common.docker;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.command.InspectContainerResponse;import com.github.dockerjava.api.model.Ports;+import io.github.hectorvent.floci.testing.TestImages;import io.quarkus.test.junit.QuarkusTest;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -20,7 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;class ContainerHostNetworkDockerIntegrationTest {private static final Logger LOG = Logger.getLogger(ContainerHostNetworkDockerIntegrationTest.class);- private static final String IMAGE = "public.ecr.aws/docker/library/busybox:1.36";+ private static final String IMAGE = TestImages.BUSYBOX;@InjectDockerClient dockerClient;diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerPlatformDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerPlatformDockerIntegrationTest.javaindex 81a12c9637..ea0e9fa7be 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerPlatformDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerPlatformDockerIntegrationTest.java@@ -4,6 +4,7 @@ import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.async.ResultCallback;import com.github.dockerjava.core.command.WaitContainerResultCallback;import com.github.dockerjava.api.model.Frame;+import io.github.hectorvent.floci.testing.TestImages;import io.quarkus.test.junit.QuarkusTest;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -23,7 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;class ContainerPlatformDockerIntegrationTest {private static final Logger LOG = Logger.getLogger(ContainerPlatformDockerIntegrationTest.class);- private static final String IMAGE = "public.ecr.aws/docker/library/busybox:1.36";+ private static final String IMAGE = TestImages.BUSYBOX_FOREIGN_PLATFORM;@InjectDockerClient dockerClient;diff --git a/src/test/java/io/github/hectorvent/floci/services/batch/BatchDockerRunnerDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/batch/BatchDockerRunnerDockerIntegrationTest.javaindex 57d6a529b4..6cfea4d525 100644--- a/src/test/java/io/github/hectorvent/floci/services/batch/BatchDockerRunnerDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/batch/BatchDockerRunnerDockerIntegrationTest.java@@ -6,6 +6,7 @@ import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;import io.github.hectorvent.floci.core.common.docker.ContainerDetector;import io.github.hectorvent.floci.services.batch.model.BatchJob;import io.github.hectorvent.floci.services.batch.model.BatchRunResult;+import io.github.hectorvent.floci.testing.TestImages;import io.quarkus.test.junit.QuarkusTest;import jakarta.inject.Inject;import org.junit.jupiter.api.Assumptions;@@ -19,7 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;@QuarkusTestclass BatchDockerRunnerDockerIntegrationTest {- private static final String IMAGE = "public.ecr.aws/docker/library/busybox:latest";+ private static final String IMAGE = TestImages.BUSYBOX;@InjectBatchDockerRunner runner;diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryDockerIntegrationTest.javaindex a483b756cb..a7a156678d 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceDiscoveryDockerIntegrationTest.java@@ -12,6 +12,7 @@ import io.github.hectorvent.floci.services.ecs.model.NetworkConfiguration;import io.github.hectorvent.floci.services.ecs.model.NetworkMode;import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;import io.github.hectorvent.floci.services.ecs.model.UpdateServiceRequest;+import io.github.hectorvent.floci.testing.TestImages;import io.quarkus.test.junit.QuarkusTest;import io.quarkus.test.junit.QuarkusTestProfile;import io.quarkus.test.junit.TestProfile;@@ -43,7 +44,7 @@ class EcsServiceDiscoveryDockerIntegrationTest {private static final String REGION = "us-east-1";private static final String SUBNET = "subnet-default-us-east-1-a";- private static final String BUSYBOX_IMAGE = "public.ecr.aws/docker/library/busybox:latest";+ private static final String BUSYBOX_IMAGE = TestImages.BUSYBOX;private static final String CONTAINER_NAME = "app";private static final String CLOUD_MAP_SERVICE_NAME = "cache";private static final String ECS_SERVICE_NAME = "cache-svc";diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsIsolationDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsIsolationDockerIntegrationTest.javaindex e83b078086..bdd6b09854 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsIsolationDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerEfsIsolationDockerIntegrationTest.java@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.services.ecs.model.EfsVolumeConfiguration;import io.github.hectorvent.floci.services.ecs.model.MountPoint;import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;import io.github.hectorvent.floci.services.ecs.model.Volume;+import io.github.hectorvent.floci.testing.TestImages;import io.quarkus.test.junit.QuarkusTest;import jakarta.inject.Inject;import org.junit.jupiter.api.AfterEach;@@ -33,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;@QuarkusTestclass EcsContainerManagerEfsIsolationDockerIntegrationTest {- private static final String IMAGE = "public.ecr.aws/docker/library/busybox:latest";+ private static final String IMAGE = TestImages.BUSYBOX;@InjectEcsContainerManager containerManager;diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensDockerIntegrationTest.javaindex d1e5226a38..8ffae7737a 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensDockerIntegrationTest.java@@ -10,6 +10,7 @@ import io.github.hectorvent.floci.services.ecs.model.EcsTask;import io.github.hectorvent.floci.services.ecs.model.FirelensConfiguration;import io.github.hectorvent.floci.services.ecs.model.LogConfiguration;import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;+import io.github.hectorvent.floci.testing.TestImages;import io.quarkus.test.junit.QuarkusTest;import jakarta.inject.Inject;import org.junit.jupiter.api.AfterEach;@@ -33,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;@QuarkusTestclass EcsContainerManagerFirelensDockerIntegrationTest {- private static final String IMAGE = "public.ecr.aws/docker/library/busybox:latest";+ private static final String IMAGE = TestImages.BUSYBOX;/** A real FireLens router image: Floci writes the generated config to its own* {@code /fluent-bit/etc}, which no minimal image carries. */private static final String ROUTER_IMAGE = "public.ecr.aws/aws-observability/aws-for-fluent-bit:3";diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerStatsDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerStatsDockerIntegrationTest.javaindex 11160dd97c..730dff7e86 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerStatsDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerStatsDockerIntegrationTest.java@@ -5,6 +5,7 @@ import com.github.dockerjava.api.model.Statistics;import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;import io.github.hectorvent.floci.services.ecs.model.EcsTask;import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;+import io.github.hectorvent.floci.testing.TestImages;import io.quarkus.test.junit.QuarkusTest;import jakarta.inject.Inject;import org.junit.jupiter.api.Assumptions;@@ -29,7 +30,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;@QuarkusTestclass EcsContainerManagerStatsDockerIntegrationTest {- private static final String BUSYBOX_IMAGE = "public.ecr.aws/docker/library/busybox:latest";+ private static final String BUSYBOX_IMAGE = TestImages.BUSYBOX;@InjectEcsContainerManager containerManager;diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromDockerIntegrationTest.javaindex 772f4df71b..533620c5e4 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromDockerIntegrationTest.java@@ -11,6 +11,7 @@ import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;import io.github.hectorvent.floci.services.ecs.model.EcsTask;import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;import io.github.hectorvent.floci.services.ecs.model.VolumeFrom;+import io.github.hectorvent.floci.testing.TestImages;import io.quarkus.test.junit.QuarkusTest;import jakarta.inject.Inject;import org.apache.commons.compress.archivers.tar.TarArchiveEntry;@@ -40,7 +41,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;@QuarkusTestclass EcsContainerManagerVolumesFromDockerIntegrationTest {- private static final String BUSYBOX_IMAGE = "public.ecr.aws/docker/library/busybox:latest";+ private static final String BUSYBOX_IMAGE = TestImages.BUSYBOX;@InjectEcsContainerManager containerManager;@@ -110,11 +111,11 @@ class EcsContainerManagerVolumesFromDockerIntegrationTest {private void buildSourceImage(String tag) throws Exception {String dockerfile = """- FROM public.ecr.aws/docker/library/busybox:latest+ FROM %sRUN mkdir -p /shared && printf '#!/bin/sh\\nexec "$@"\\n' > /shared/wrapper \&& chmod +x /shared/wrapperVOLUME ["/shared"]- """;+ """.formatted(TestImages.BUSYBOX);byte[] dockerfileBytes = dockerfile.getBytes(StandardCharsets.UTF_8);ByteArrayOutputStream context = new ByteArrayOutputStream();diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsCredentialsProxyDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsCredentialsProxyDockerIntegrationTest.javaindex d30a272bed..a6ba86c78e 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsCredentialsProxyDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsCredentialsProxyDockerIntegrationTest.java@@ -13,6 +13,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;import io.github.hectorvent.floci.services.iam.IamService;+import io.github.hectorvent.floci.testing.TestImages;import io.quarkus.test.junit.QuarkusTest;import io.vertx.core.Vertx;import jakarta.inject.Inject;@@ -152,7 +153,7 @@ class EcsCredentialsProxyDockerIntegrationTest {// kernel routes toward the default gateway instead of ARPing the peer directly, which// never reaches the proxy's socket. The real allocation (unique per task, avoiding the// proxy's own .2) is EcsContainerManager's job; this just proves the proxy's own side.- ContainerSpec taskSpec = containerBuilder.newContainer("public.ecr.aws/docker/library/python:3.12-alpine")+ ContainerSpec taskSpec = containerBuilder.newContainer(TestImages.PYTHON_ALPINE).withName("floci-ecs-credentials-proxy-test-task-" + suffix).withDockerNetwork(Optional.of(network)).withLinkLocalIp("169.254.170.5")diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/exec/EcsExecChannelDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/exec/EcsExecChannelDockerIntegrationTest.javaindex f6f386051c..f2703c8e27 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/exec/EcsExecChannelDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/exec/EcsExecChannelDockerIntegrationTest.java@@ -6,6 +6,7 @@ import io.github.hectorvent.floci.services.ecs.container.EcsTaskHandle;import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;import io.github.hectorvent.floci.services.ecs.model.EcsTask;import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;+import io.github.hectorvent.floci.testing.TestImages;import io.quarkus.test.junit.QuarkusTest;import io.restassured.RestAssured;import io.vertx.core.Vertx;@@ -39,7 +40,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;@QuarkusTestclass EcsExecChannelDockerIntegrationTest {- private static final String BUSYBOX_IMAGE = "public.ecr.aws/docker/library/busybox:latest";+ private static final String BUSYBOX_IMAGE = TestImages.BUSYBOX;private static final String TASK_ARN ="arn:aws:ecs:us-east-1:000000000000:task/exec-cluster/execdockertask";diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.javaindex eccce01533..8d51c1ba13 100644--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerDockerIntegrationTest.java@@ -6,6 +6,7 @@ import com.github.dockerjava.api.model.Event;import com.github.dockerjava.api.model.EventType;import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.github.hectorvent.floci.testing.TestImages;import io.quarkus.test.junit.QuarkusTest;import jakarta.inject.Inject;import org.junit.jupiter.api.Assumptions;@@ -59,7 +60,7 @@ class SageMakerDockerIntegrationTest {{"TrainingJobName":"%s","AlgorithmSpecification":{- "TrainingImage":"public.ecr.aws/docker/library/busybox:stable",+ "TrainingImage":"%s","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":["mkdir -p /opt/ml/model && cp /opt/ml/input/data/train/data.txt /opt/ml/model/model.txt"]},@@ -68,7 +69,7 @@ class SageMakerDockerIntegrationTest {"ResourceConfig":{"InstanceType":"ml.m5.large","InstanceCount":1,"VolumeSizeInGB":1},"StoppingCondition":{"MaxRuntimeInSeconds":60}}- """.formatted(job, bucket, bucket)).then().statusCode(200).body("TrainingJobArn", notNullValue());+ """.formatted(job, TestImages.BUSYBOX, bucket, bucket)).then().statusCode(200).body("TrainingJobArn", notNullValue());waitForTraining(job, "Completed");String artifact = post("SageMaker.DescribeTrainingJob", "{\"TrainingJobName\":\"%s\"}".formatted(job))@@ -86,8 +87,8 @@ class SageMakerDockerIntegrationTest {String endpoint = "sm-endpoint-" + suffix;String script = "trap 'exit 0' TERM; cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200 if self.path == '/ping' else 404); self.end_headers()\n def do_POST(self):\n n=int(self.headers.get('content-length','0')); b=self.rfile.read(n); self.send_response(200); self.send_header('Content-Type','text/plain'); self.end_headers(); self.wfile.write(b.upper())\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py & wait $!";post("SageMaker.CreateModel", """- {"ModelName":"%s","PrimaryContainer":{"Image":"public.ecr.aws/docker/library/python:3-alpine","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}- """.formatted(model, json(script))).then().statusCode(200);+ {"ModelName":"%s","PrimaryContainer":{"Image":"%s","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}+ """.formatted(model, TestImages.PYTHON_ALPINE, json(script))).then().statusCode(200);post("SageMaker.CreateEndpointConfig", """{"EndpointConfigName":"%s","ProductionVariants":[{"VariantName":"AllTraffic","ModelName":"%s","InitialInstanceCount":1,"InstanceType":"ml.t2.medium","InitialVariantWeight":1.0}]}""".formatted(cfg, model)).then().statusCode(200);@@ -111,7 +112,7 @@ class SageMakerDockerIntegrationTest {{"TrainingJobName":"%s","AlgorithmSpecification":{- "TrainingImage":"public.ecr.aws/docker/library/busybox:stable",+ "TrainingImage":"%s","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":["trap 'exit 143' TERM; mkfifo /tmp/hold; cat /tmp/hold & wait"]},@@ -119,7 +120,7 @@ class SageMakerDockerIntegrationTest {"ResourceConfig":{"InstanceType":"ml.m5.large","InstanceCount":1,"VolumeSizeInGB":1},"StoppingCondition":{"MaxRuntimeInSeconds":300}}- """.formatted(job, bucket)).then().statusCode(200);+ """.formatted(job, TestImages.BUSYBOX, bucket)).then().statusCode(200);// Wait for the container to actually be running, blocked on an empty pipe, so the stop// races a real running container rather than one still being staged.awaitContainerRunning("floci-aws-sagemaker-training-" + job);@@ -144,8 +145,8 @@ class SageMakerDockerIntegrationTest {// up, well before /ping can succeed, so the start it races is always still in flight.String script = "trap 'exit 0' TERM; cat > /server.py <<'PY'\nfrom http.server import BaseHTTPRequestHandler,HTTPServer\nclass H(BaseHTTPRequestHandler):\n def do_GET(self):\n self.send_response(200); self.end_headers()\nHTTPServer(('0.0.0.0',8080),H).serve_forever()\nPY\npython /server.py & wait $!";post("SageMaker.CreateModel", """- {"ModelName":"%s","PrimaryContainer":{"Image":"public.ecr.aws/docker/library/python:3-alpine","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}- """.formatted(model, json(script))).then().statusCode(200);+ {"ModelName":"%s","PrimaryContainer":{"Image":"%s","ContainerEntrypoint":["/bin/sh","-c"],"ContainerArguments":[%s]}}+ """.formatted(model, TestImages.PYTHON_ALPINE, json(script))).then().statusCode(200);post("SageMaker.CreateEndpointConfig", """{"EndpointConfigName":"%s","ProductionVariants":[{"VariantName":"AllTraffic","ModelName":"%s","InitialInstanceCount":1,"InstanceType":"ml.t2.medium","InitialVariantWeight":1.0}]}""".formatted(cfg, model)).then().statusCode(200);diff --git a/src/test/java/io/github/hectorvent/floci/testing/TestImages.java b/src/test/java/io/github/hectorvent/floci/testing/TestImages.javanew file mode 100644index 0000000000..e10c63edab--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/testing/TestImages.java@@ -0,0 +1,31 @@+package io.github.hectorvent.floci.testing;++/**+ * Images the Docker-backed tests start, one constant per image so a tag lives in one place.+ * {@code .github/ci/prefetch-images.sh} reads the values from this file, the way it reads the+ * sidecar pins from {@code application.yml}, so the prefetch cannot drift from the tests.+ * <p>+ * The Docker library images are named the Docker Hub way, not through the+ * {@code public.ecr.aws/docker/library} mirror. That mirror has an anonymous quota of its own,+ * shared by runner IP and often spent before a job starts: on one CI job the first pull of the+ * job, {@code docker/library/busybox} from that mirror, failed with+ * {@code toomanyrequests: Data limit exceeded}, while the same runner pulled {@code busybox} from+ * Docker Hub seven seconds later and a 116 MiB {@code lambda/} image from ECR Public half a+ * minute after that. GitHub-hosted runners pull Docker Hub without a limit.+ */+public final class TestImages {++ public static final String BUSYBOX = "busybox:stable";++ /**+ * For the one test that pulls the other architecture. A foreign-platform pull replaces what+ * the local tag points at on a daemon without the containerd image store, so it must not share+ * a tag with the tests that run or build from {@link #BUSYBOX} on the host architecture.+ */+ public static final String BUSYBOX_FOREIGN_PLATFORM = "busybox:1.36";++ public static final String PYTHON_ALPINE = "python:3.12-alpine";++ private TestImages() {+ }+}