Light, fluffy, and always free - The AWS Local Emulator alternative
fix(docker): retry core and EKS archive uploads (#5916)
Aniket Shuklad951fd532cd5
diff --git a/docs/configuration/docker.md b/docs/configuration/docker.mdindex f70f50e9ef..72e1d2195d 100644--- a/docs/configuration/docker.md+++ b/docs/configuration/docker.md@@ -322,6 +322,13 @@ All of Floci's short-lived docker calls (create/start/inspect/remove container,A request is only replayed when doing so cannot change semantics: requests carrying a one-shot upload stream (e.g. copying an archive into a container), bidirectional attach streams, and `exec` requests (which would re-run the command) are never retried. Log-follow connections use a separate transport that does not retry at all. Bodyless actions whose replay changes the outcome (unnamed container create, network create, container restart, network connect and disconnect) are not retried either. A tar-writer failure on the Floci host (for example an unreadable source file) is reported at once instead of retried.+Archive uploads that use the shared tar-copy helper retry above the transport layer, rebuilding+the upload stream for each attempt with the same six-attempt backoff. This includes user-data+scripts, the container CA bundle, and EKS audit policies, IRSA signing keys, the containerd+certificate-directory link and the Pod Identity manifest. Only the upload is retried, not+script execution. Daemon rejections still stop immediately, and each caller keeps its existing+failure handling after retries are exhausted.+## Full Reference| Environment variable | Default | Description |diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javaindex 4fee6050d6..ca4b409d2d 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -30,12 +30,8 @@ import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.Lauimport io.quarkus.runtime.annotations.RegisterForReflection;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;import org.jboss.logging.Logger;-import java.io.ByteArrayInputStream;-import java.io.ByteArrayOutputStream;import java.io.Closeable;import java.nio.file.Files;import java.nio.file.Path;@@ -91,6 +87,7 @@ public class ContainerLifecycleManager {private final ImageCacheService imageCacheService;private final ContainerDetector containerDetector;private final EmulatorConfig config;+ private final long archiveRetryBackoffMillis;/** Volumes whose shared-ownership root has already been initialised this process (run-once guard). */private final ConcurrentHashMap<String, Boolean> initializedSharedVolumes = new ConcurrentHashMap<>();@@ -103,10 +100,20 @@ public class ContainerLifecycleManager {ImageCacheService imageCacheService,ContainerDetector containerDetector,EmulatorConfig config) {+ this(dockerClient, imageCacheService, containerDetector, config,+ RetryingTarCopier.DEFAULT_BACKOFF_MS);+ }++ ContainerLifecycleManager(DockerClient dockerClient,+ ImageCacheService imageCacheService,+ ContainerDetector containerDetector,+ EmulatorConfig config,+ long archiveRetryBackoffMillis) {this.dockerClient = dockerClient;this.imageCacheService = imageCacheService;this.containerDetector = containerDetector;this.config = config;+ this.archiveRetryBackoffMillis = archiveRetryBackoffMillis;}/**@@ -331,18 +338,9 @@ public class ContainerLifecycleManager {private boolean copyCaBundle(String containerId, Path bundle) {try {byte[] content = Files.readAllBytes(bundle);- ByteArrayOutputStream archive = new ByteArrayOutputStream(content.length + 1024);- try (TarArchiveOutputStream tar = new TarArchiveOutputStream(archive)) {- TarArchiveEntry entry = new TarArchiveEntry(ContainerCaBundle.FILE_NAME);- entry.setSize(content.length);- tar.putArchiveEntry(entry);- tar.write(content);- tar.closeArchiveEntry();- }- dockerClient.copyArchiveToContainerCmd(containerId)- .withRemotePath(ContainerCaBundle.CONTAINER_DIR)- .withTarInputStream(new ByteArrayInputStream(archive.toByteArray()))- .exec();+ RetryingTarCopier.copyBytes(dockerClient, containerId, ContainerCaBundle.CONTAINER_DIR,+ ContainerCaBundle.FILE_NAME, content, 0644,+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);LOG.debugv("Copied the CA bundle into container {0} at {1}", containerId, ContainerCaBundle.CONTAINER_PATH);return true;} catch (Exception e) {diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.javaindex 1f44874586..aec79c55ad 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.core.common.docker;import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.exception.DockerClientException;import org.apache.commons.compress.archivers.tar.TarArchiveEntry;import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;import org.jboss.logging.Logger;@@ -30,8 +31,8 @@ public final class RetryingTarCopier {private static final Logger LOG = Logger.getLogger(RetryingTarCopier.class);- static final int DEFAULT_MAX_ATTEMPTS = 6;- static final long DEFAULT_BACKOFF_MS = 500L;+ public static final int DEFAULT_MAX_ATTEMPTS = 6;+ public static final long DEFAULT_BACKOFF_MS = 500L;/*** Buffer for the tar-streaming pipe. The default {@link PipedInputStream} buffer is only 1KB,@@ -154,11 +155,22 @@ public final class RetryingTarCopier {public static void copyHostResource(DockerClient docker, String containerId, String remotePath,String hostPath, int maxAttempts, long backoffMillis) {- retry(hostPath, containerId, maxAttempts, backoffMillis, () ->+ retry(hostPath, containerId, maxAttempts, backoffMillis, () -> {+ try {docker.copyArchiveToContainerCmd(containerId).withHostResource(hostPath).withRemotePath(remotePath)- .exec());+ .exec();+ } catch (DockerClientException e) {+ // These docker-java wrappers identify local archive preparation and temporary-file I/O.+ String message = e.getMessage();+ if (message != null && (message.startsWith("Unable to perform tar on host resource ")+ || message.startsWith("Unable to read temp file "))) {+ throw new DockerRetry.LocalFailure(message, e);+ }+ throw e;+ }+ });}/** Single-entry tar bytes, reusable across attempts and callers. */diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/UserDataPipeline.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/UserDataPipeline.javaindex ba18990805..136cf83f3b 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/UserDataPipeline.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/UserDataPipeline.java@@ -3,8 +3,6 @@ package io.github.hectorvent.floci.core.common.docker;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.async.ResultCallback;import com.github.dockerjava.api.model.Frame;-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;import org.jboss.logging.Logger;import java.io.ByteArrayInputStream;@@ -203,20 +201,6 @@ public final class UserDataPipeline {return false;}- public static byte[] buildSingleFileTar(String filename, byte[] content, int mode) throws IOException {- ByteArrayOutputStream bos = new ByteArrayOutputStream();- try (TarArchiveOutputStream tar = new TarArchiveOutputStream(bos)) {- tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);- TarArchiveEntry entry = new TarArchiveEntry(filename);- entry.setSize(content.length);- entry.setMode(mode);- tar.putArchiveEntry(entry);- tar.write(content);- tar.closeArchiveEntry();- }- return bos.toByteArray();- }-public static String summarizeUserDataOutput(BoundedOutput output) {if (output == null) {return "(no output)";@@ -239,6 +223,20 @@ public final class UserDataPipeline {Duration timeout,Consumer<String> outputLineConsumer,Set<ResultCallback<Frame>> activeCallbacks+ ) {+ return executeUserData(dockerClient, containerId, contextDescription, userData, timeout,+ outputLineConsumer, activeCallbacks, RetryingTarCopier.DEFAULT_BACKOFF_MS);+ }++ static ExecutionResult executeUserData(+ DockerClient dockerClient,+ String containerId,+ String contextDescription,+ String userData,+ Duration timeout,+ Consumer<String> outputLineConsumer,+ Set<ResultCallback<Frame>> activeCallbacks,+ long archiveRetryBackoffMillis) {if (dockerClient == null || containerId == null || containerId.isBlank()) {return ExecutionResult.skipped("No container available");@@ -255,7 +253,7 @@ public final class UserDataPipeline {String scriptContent = shellScripts.get(i);ExecutionResult partResult = executeSingleScript(dockerClient, containerId, contextDescription, scriptContent, partNumber, partCount,- effectiveTimeout, outputLineConsumer, activeCallbacks);+ effectiveTimeout, outputLineConsumer, activeCallbacks, archiveRetryBackoffMillis);if (!partResult.isSuccess()) {return partResult;}@@ -272,15 +270,13 @@ public final class UserDataPipeline {int partCount,Duration timeout,Consumer<String> outputLineConsumer,- Set<ResultCallback<Frame>> activeCallbacks+ Set<ResultCallback<Frame>> activeCallbacks,+ long archiveRetryBackoffMillis) {byte[] script = scriptContent.getBytes(StandardCharsets.UTF_8);try {- byte[] tar = buildSingleFileTar("user-data.sh", script, 0755);- dockerClient.copyArchiveToContainerCmd(containerId)- .withRemotePath("/var/lib")- .withTarInputStream(new ByteArrayInputStream(tar))- .exec();+ RetryingTarCopier.copyBytes(dockerClient, containerId, "/var/lib", "user-data.sh", script, 0755,+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);String execId = dockerClient.execCreateCmd(containerId).withCmd(USER_DATA_SCRIPT_PATH)diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.javaindex 4b25652e0d..5de58680a1 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java@@ -62,7 +62,6 @@ import org.apache.commons.compress.archivers.tar.TarArchiveEntry;import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;import org.jboss.logging.Logger;-import java.io.ByteArrayInputStream;import java.io.ByteArrayOutputStream;import java.io.Closeable;import java.io.File;@@ -158,6 +157,7 @@ public class EksClusterManagerprivate final ContainerBuilder containerBuilder;private final ContainerLifecycleManager lifecycleManager;+ private final long archiveRetryBackoffMillis;private final ContainerDetector containerDetector;private final PortAllocator portAllocator;private final DockerHostResolver dockerHostResolver;@@ -424,6 +424,26 @@ public class EksClusterManagerFlociCertificateAuthority certificateAuthority,ContainerLogStreamer logStreamer,StorageBackend<String, Nodegroup> nodeGroupStorage) {+ this(containerBuilder, lifecycleManager, containerDetector, portAllocator,+ dockerHostResolver, ecrRegistryManager, config, regionResolver, metadataServer,+ oidcService, certificateAuthority, logStreamer, nodeGroupStorage,+ RetryingTarCopier.DEFAULT_BACKOFF_MS);+ }++ EksClusterManager(ContainerBuilder containerBuilder,+ ContainerLifecycleManager lifecycleManager,+ ContainerDetector containerDetector,+ PortAllocator portAllocator,+ DockerHostResolver dockerHostResolver,+ EcrRegistryManager ecrRegistryManager,+ EmulatorConfig config,+ RegionResolver regionResolver,+ Ec2MetadataServer metadataServer,+ EksOidcService oidcService,+ FlociCertificateAuthority certificateAuthority,+ ContainerLogStreamer logStreamer,+ StorageBackend<String, Nodegroup> nodeGroupStorage,+ long archiveRetryBackoffMillis) {this.containerBuilder = containerBuilder;this.lifecycleManager = lifecycleManager;this.containerDetector = containerDetector;@@ -437,6 +457,7 @@ public class EksClusterManagerthis.certificateAuthority = certificateAuthority;this.logStreamer = logStreamer;this.nodeGroupStorage = nodeGroupStorage;+ this.archiveRetryBackoffMillis = archiveRetryBackoffMillis;}@Inject@@ -1554,7 +1575,8 @@ public class EksClusterManagerprivate void copyWebhookIntoContainer(String containerId, String localFile, String clusterName) {try {RetryingTarCopier.copyHostResource(lifecycleManager.getDockerClient(), containerId,- WEBHOOK_CONFIG_DIR, localFile);+ WEBHOOK_CONFIG_DIR, localFile,+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);} catch (Exception e) {LOG.warnv("EKS token-webhook may not authenticate for cluster {0}: could not copy kubeconfig "+ "into the k3s container: {1}", clusterName, e.getMessage());@@ -1592,10 +1614,8 @@ public class EksClusterManagertry {DockerClient dockerClient = lifecycleManager.getDockerClient();if (dockerClient != null) {- dockerClient.copyArchiveToContainerCmd(containerId)- .withHostResource(localFile)- .withRemotePath(AUDIT_POLICY_DIR)- .exec();+ RetryingTarCopier.copyHostResource(dockerClient, containerId, AUDIT_POLICY_DIR, localFile,+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);LOG.debugv("Injected audit policy file into k3s container {0} for cluster {1}",containerId, clusterName);}@@ -1919,16 +1939,12 @@ public class EksClusterManager*/void copySigningKeysIntoContainer(String containerId, SigningKeyFiles keyFiles, String clusterName) {try {- lifecycleManager.getDockerClient()- .copyArchiveToContainerCmd(containerId)- .withHostResource(keyFiles.signingKeyPath().toString())- .withRemotePath(WEBHOOK_CONFIG_DIR)- .exec();- lifecycleManager.getDockerClient()- .copyArchiveToContainerCmd(containerId)- .withHostResource(keyFiles.publicKeyPath().toString())- .withRemotePath(WEBHOOK_CONFIG_DIR)- .exec();+ RetryingTarCopier.copyHostResource(lifecycleManager.getDockerClient(), containerId,+ WEBHOOK_CONFIG_DIR, keyFiles.signingKeyPath().toString(),+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);+ RetryingTarCopier.copyHostResource(lifecycleManager.getDockerClient(), containerId,+ WEBHOOK_CONFIG_DIR, keyFiles.publicKeyPath().toString(),+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);LOG.debugv("Injected IRSA OIDC signing keypair into k3s container {0} for cluster {1}",containerId, clusterName);} catch (Exception e) {@@ -1983,7 +1999,8 @@ public class EksClusterManager"registries.yaml"), content, clusterName);try {RetryingTarCopier.copyBytes(lifecycleManager.getDockerClient(), containerId, "/etc",- REGISTRIES_TAR_ENTRY, content.getBytes(StandardCharsets.UTF_8), 0644);+ REGISTRIES_TAR_ENTRY, content.getBytes(StandardCharsets.UTF_8), 0644,+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);LOG.infov("Injected ECR registry mirror ({0}) into k3s cluster {1}", endpoint, clusterName);} catch (Exception e) {LOG.warnv("EKS cluster {0} gets no ECR registry mirror: could not copy registries.yaml "@@ -2001,11 +2018,10 @@ public class EksClusterManager*/void linkContainerdCertsDir(String containerId, String clusterName) {try {- lifecycleManager.getDockerClient()- .copyArchiveToContainerCmd(containerId)- .withTarInputStream(new ByteArrayInputStream(buildContainerdCertsLinkTar()))- .withRemotePath("/")- .exec();+ byte[] tar = buildContainerdCertsLinkTar();+ RetryingTarCopier.copyStreamed(lifecycleManager.getDockerClient(), containerId, "/",+ "containerd-certs-link", out -> out.write(tar),+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);} catch (Exception e) {LOG.warnv("EKS cluster {0}: could not link containerd certs.d directory: {1}",clusterName, e.getMessage());@@ -2071,12 +2087,10 @@ public class EksClusterManagerwriteLocalCopy(Paths.get(config.services().eks().dataPath(), "webhook", clusterName,POD_IDENTITY_MANIFEST_FILE), manifest, clusterName);try {- lifecycleManager.getDockerClient()- .copyArchiveToContainerCmd(containerId)- .withTarInputStream(new ByteArrayInputStream(- tarSingleFile(POD_IDENTITY_MANIFEST_TAR_ENTRY, manifest)))- .withRemotePath(K3S_DATA_DIR)- .exec();+ byte[] tar = tarSingleFile(POD_IDENTITY_MANIFEST_TAR_ENTRY, manifest);+ RetryingTarCopier.copyStreamed(lifecycleManager.getDockerClient(), containerId, K3S_DATA_DIR,+ POD_IDENTITY_MANIFEST_FILE, out -> out.write(tar),+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);LOG.infov("Registered the EKS Pod Identity mutating webhook ({0}) for cluster {1}", url, clusterName);} catch (Exception e) {LOG.warnv("EKS cluster {0} gets no pod identity injection: could not copy {1} into the k3s "diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCaBundleTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCaBundleTest.javaindex 2f06ea318d..dce357a14c 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCaBundleTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCaBundleTest.java@@ -16,19 +16,28 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.extension.ExtendWith;import org.junit.jupiter.api.io.TempDir;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;import org.mockito.ArgumentCaptor;import org.mockito.Mock;import org.mockito.junit.jupiter.MockitoExtension;+import java.io.ByteArrayInputStream;+import java.io.IOException;import java.io.InputStream;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;+import java.util.ArrayList;import java.util.List;import java.util.Map;import java.util.Optional;+import java.util.concurrent.atomic.AtomicInteger;+import static org.junit.jupiter.api.Assertions.assertArrayEquals;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNotSame;import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;@@ -83,10 +92,19 @@ class ContainerLifecycleManagerCaBundleTest {@Testvoid tlsOnCopiesTheBundleAndAppendsTheTrustEnvAfterTheSpecEnv() throws Exception {when(tlsConfig.enabled()).thenReturn(true);- Path bundle = writeBundle();+ writeBundle();CreateContainerCmd createCmd = stubCreateContainer();CopyArchiveToContainerCmd copyCmd = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);when(dockerClient.copyArchiveToContainerCmd("container-id")).thenReturn(copyCmd);+ List<InputStream> streams = new ArrayList<>();+ when(copyCmd.withTarInputStream(any(InputStream.class))).thenAnswer(invocation -> {+ streams.add(invocation.getArgument(0));+ return copyCmd;+ });+ when(copyCmd.exec()).thenAnswer(invocation -> {+ assertBundleArchive(streams.getLast());+ return null;+ });assertEquals("container-id", manager().create(specWithEnv(List.of("FOO=bar"))));@@ -99,16 +117,49 @@ class ContainerLifecycleManagerCaBundleTest {"AWS_CA_BUNDLE=/etc/floci-ca-bundle.pem"), env);verify(copyCmd).withRemotePath("/etc");- ArgumentCaptor<InputStream> tar = ArgumentCaptor.forClass(InputStream.class);- verify(copyCmd).withTarInputStream(tar.capture());+ assertEquals(1, streams.size());verify(copyCmd).exec();- try (TarArchiveInputStream in = new TarArchiveInputStream(tar.getValue())) {- TarArchiveEntry entry = in.getNextEntry();- assertEquals("floci-ca-bundle.pem", entry.getName());- assertEquals(0644, entry.getMode() & 0777, "world-readable, so a non-root USER can read it");- assertEquals(Files.readString(bundle), new String(in.readAllBytes(), StandardCharsets.UTF_8));- assertNull(in.getNextEntry(), "the archive holds only the bundle");- }+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void transientCopyFailurePreservesTheContainerAndTrustWithAFreshTar(boolean consumeCompletely) throws Exception {+ when(tlsConfig.enabled()).thenReturn(true);+ writeBundle();+ CreateContainerCmd createCmd = stubCreateContainer();+ CopyArchiveToContainerCmd copyCmd = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);+ when(dockerClient.copyArchiveToContainerCmd("container-id")).thenReturn(copyCmd);+ List<InputStream> streams = new ArrayList<>();+ when(copyCmd.withTarInputStream(any(InputStream.class))).thenAnswer(invocation -> {+ streams.add(invocation.getArgument(0));+ return copyCmd;+ });+ AtomicInteger attempts = new AtomicInteger();+ when(copyCmd.exec()).thenAnswer(invocation -> {+ int attempt = attempts.incrementAndGet();+ InputStream stream = streams.getLast();+ if (attempt == 1 && !consumeCompletely) {+ assertEquals(513, stream.readNBytes(513).length, "consume the tar header and part of the CA bundle");+ } else {+ assertBundleArchive(stream);+ }+ if (attempt == 1) {+ throw new RuntimeException(new IOException("Connection reset"));+ }+ return null;+ });+ lenient().when(dockerClient.removeContainerCmd("container-id"))+ .thenReturn(mock(RemoveContainerCmd.class, RETURNS_SELF));++ assertEquals("container-id", manager().create(specWithEnv(List.of("FOO=bar"))));++ assertEquals(2, attempts.get());+ assertEquals(2, streams.size());+ assertNotSame(streams.getFirst(), streams.get(1), "a consumed tar stream cannot be replayed");+ assertTrue(capturedEnv(createCmd).contains("AWS_CA_BUNDLE=/etc/floci-ca-bundle.pem"));+ verify(copyCmd, times(2)).withRemotePath("/etc");+ verify(createCmd).exec();+ verify(dockerClient, never()).removeContainerCmd(anyString());}@Test@@ -211,8 +262,20 @@ class ContainerLifecycleManagerCaBundleTest {return Files.writeString(tlsDir.resolve("floci-ca-bundle.pem"), BUNDLE_PEM);}+ private static void assertBundleArchive(InputStream stream) throws IOException {+ byte[] archive = stream.readAllBytes();+ try (TarArchiveInputStream in = new TarArchiveInputStream(new ByteArrayInputStream(archive))) {+ TarArchiveEntry entry = in.getNextEntry();+ assertNotNull(entry, "every attempt must begin with a complete tar header");+ assertEquals("floci-ca-bundle.pem", entry.getName());+ assertEquals(0644, entry.getMode() & 0777, "world-readable, so a non-root USER can read it");+ assertArrayEquals(BUNDLE_PEM.getBytes(StandardCharsets.UTF_8), in.readAllBytes());+ assertNull(in.getNextEntry(), "the archive holds only the bundle");+ }+ }+private ContainerLifecycleManager manager() {- return new ContainerLifecycleManager(dockerClient, imageCacheService, containerDetector, config);+ return new ContainerLifecycleManager(dockerClient, imageCacheService, containerDetector, config, 0L);}private static ContainerSpec specWithEnv(List<String> env) {diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.javaindex a5eaac12af..272a45139c 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.java@@ -2,34 +2,55 @@ package io.github.hectorvent.floci.core.common.docker;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;+import com.github.dockerjava.api.exception.DockerClientException;+import com.github.dockerjava.api.exception.NotFoundException;+import com.github.dockerjava.core.DefaultDockerClientConfig;+import com.github.dockerjava.core.DockerClientImpl;+import com.github.dockerjava.core.util.CompressArchiveUtil;+import com.github.dockerjava.transport.DockerHttpClient;import org.apache.commons.compress.archivers.tar.TarArchiveEntry;import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.io.TempDir;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.NullSource;+import org.junit.jupiter.params.provider.ValueSource;import org.mockito.ArgumentCaptor;+import org.mockito.MockMakers;+import org.mockito.MockedStatic;import java.io.ByteArrayInputStream;import java.io.IOException;import java.io.InputStream;import java.nio.charset.StandardCharsets;+import java.nio.file.FileSystemException;import java.nio.file.Files;import java.nio.file.NoSuchFileException;import java.nio.file.Path;+import java.util.Map;import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.atomic.AtomicReference;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertInstanceOf;import static org.junit.jupiter.api.Assertions.assertNotSame;import static org.junit.jupiter.api.Assertions.assertSame;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyBoolean;+import static org.mockito.Mockito.CALLS_REAL_METHODS;import static org.mockito.Mockito.RETURNS_SELF;import static org.mockito.Mockito.atLeastOnce;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.mockStatic;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.times;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;+import static org.mockito.Mockito.withSettings;/*** The transport-level retry ({@code RetryingDockerHttpClient}) must refuse to replay a one-shot@@ -47,6 +68,13 @@ class RetryingTarCopierTest {return new RuntimeException(new IOException("Broken pipe"));}+ private static DockerClient dockerClient(DockerHttpClient transport) {+ return DockerClientImpl.getInstance(DefaultDockerClientConfig.createDefaultConfigBuilder()+ .withDockerHost("tcp://localhost:2375")+ .withApiVersion("1.44")+ .build(), transport);+ }+private static CopyArchiveToContainerCmd failingOnce(DockerClient docker, AtomicInteger calls) {CopyArchiveToContainerCmd cmd = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);when(cmd.exec()).thenAnswer(inv -> {@@ -180,14 +208,128 @@ class RetryingTarCopierTest {void copyHostResourceRetriesTransientBrokenPipe(@TempDir Path dir) throws IOException {Path file = dir.resolve("kubeconfig.yaml");Files.writeString(file, "apiVersion: v1");- DockerClient docker = mock(DockerClient.class);- AtomicInteger calls = new AtomicInteger();- failingOnce(docker, calls);+ DockerHttpClient transport = mock(DockerHttpClient.class);+ DockerHttpClient.Response ok = mock(DockerHttpClient.Response.class);+ when(ok.getStatusCode()).thenReturn(200);+ when(transport.execute(any())).thenThrow(brokenPipe()).thenReturn(ok);- RetryingTarCopier.copyHostResource(docker, "c-1", "/var/lib/rancher", file.toString(),- MAX_ATTEMPTS, 0L);+ try (DockerClient docker = dockerClient(transport)) {+ RetryingTarCopier.copyHostResource(docker, "c-1", "/var/lib/rancher", file.toString(),+ MAX_ATTEMPTS, 0L);+ }- assertEquals(2, calls.get());+ verify(transport, times(2)).execute(any());+ }++ @ParameterizedTest+ @NullSource+ @ValueSource(strings = {"Docker transport failure"})+ void copyHostResourceStillRetriesTransportClientExceptions(String message, @TempDir Path dir) throws IOException {+ Path file = dir.resolve("kubeconfig.yaml");+ Files.writeString(file, "apiVersion: v1");+ DockerHttpClient transport = mock(DockerHttpClient.class);+ DockerHttpClient.Response ok = mock(DockerHttpClient.Response.class);+ when(ok.getStatusCode()).thenReturn(200);+ when(transport.execute(any()))+ .thenThrow(new DockerClientException(message, new IOException("Broken pipe"))).thenReturn(ok);++ try (DockerClient docker = dockerClient(transport)) {+ RetryingTarCopier.copyHostResource(docker, "c-1", "/var/lib/rancher", file.toString(),+ MAX_ATTEMPTS, 0L);+ }++ verify(transport, times(2)).execute(any());+ }++ @Test+ void copyHostResourceDoesNotRetryLocalArchiveFailure(@TempDir Path dir) throws IOException {+ Path file = dir.resolve("kubeconfig.yaml");+ Files.writeString(file, "apiVersion: v1");+ DockerHttpClient transport = mock(DockerHttpClient.class);+ IOException archiveFailure = new IOException("No space left on device");+ AtomicInteger archiveAttempts = new AtomicInteger();++ try (DockerClient docker = dockerClient(transport);+ MockedStatic<CompressArchiveUtil> archives = mockStatic(CompressArchiveUtil.class,+ withSettings().mockMaker(MockMakers.INLINE).defaultAnswer(CALLS_REAL_METHODS))) {+ archives.when(() -> CompressArchiveUtil.tar(any(Path.class), any(Path.class),+ anyBoolean(), anyBoolean())).thenAnswer(invocation -> {+ archiveAttempts.incrementAndGet();+ throw archiveFailure;+ });++ RuntimeException thrown = assertThrows(RuntimeException.class,+ () -> RetryingTarCopier.copyHostResource(docker, "c-1", "/var/lib/rancher",+ file.toString(), MAX_ATTEMPTS, 0L));++ Throwable cause = thrown;+ while (cause.getCause() != null) {+ cause = cause.getCause();+ }+ assertSame(archiveFailure, cause, "the original local archive failure must survive");+ verify(transport, never()).execute(any());+ assertEquals(1, archiveAttempts.get(), "local archive failures cannot recover by retrying Docker");+ }+ }++ @Test+ void copyHostResourceDoesNotRetryLocalArchiveReadFailure(@TempDir Path dir) throws IOException {+ Path file = dir.resolve("kubeconfig.yaml");+ Files.writeString(file, "apiVersion: v1");+ DockerHttpClient transport = mock(DockerHttpClient.class);+ AtomicInteger archiveAttempts = new AtomicInteger();+ AtomicReference<Path> archivePath = new AtomicReference<>();++ try (DockerClient docker = dockerClient(transport);+ MockedStatic<CompressArchiveUtil> archives = mockStatic(CompressArchiveUtil.class,+ withSettings().mockMaker(MockMakers.INLINE).defaultAnswer(CALLS_REAL_METHODS))) {+ archives.when(() -> CompressArchiveUtil.tar(any(Path.class), any(Path.class),+ anyBoolean(), anyBoolean())).thenAnswer(invocation -> {+ archiveAttempts.incrementAndGet();+ Path archive = invocation.getArgument(1);+ archivePath.set(archive);+ Files.delete(archive);+ // A symlink loop fails when docker-java opens the archive, before HTTP.+ Files.createSymbolicLink(archive, archive.getFileName());+ return null;+ });++ RuntimeException thrown = assertThrows(RuntimeException.class,+ () -> RetryingTarCopier.copyHostResource(docker, "c-1", "/var/lib/rancher",+ file.toString(), MAX_ATTEMPTS, 0L));++ Throwable cause = thrown;+ while (cause.getCause() != null) {+ cause = cause.getCause();+ }+ FileSystemException readFailure = assertInstanceOf(FileSystemException.class, cause);+ assertEquals(archivePath.get().toString(), readFailure.getFile());+ verify(transport, never()).execute(any());+ assertEquals(1, archiveAttempts.get(), "opening a local archive must not spend the Docker retry budget");+ }+ }++ @Test+ void copyHostResourceDoesNotRetryDaemonRejection(@TempDir Path dir) throws IOException {+ Path file = dir.resolve("kubeconfig.yaml");+ Files.writeString(file, "apiVersion: v1");+ DockerHttpClient transport = mock(DockerHttpClient.class);+ DockerHttpClient.Response missingContainer = mock(DockerHttpClient.Response.class);+ when(missingContainer.getStatusCode()).thenReturn(404);+ when(missingContainer.getHeaders()).thenReturn(Map.of());+ when(missingContainer.getBody()).thenAnswer(invocation -> new ByteArrayInputStream(+ "{\"message\":\"No such container: c-1\"}".getBytes(StandardCharsets.UTF_8)));+ when(transport.execute(any())).thenReturn(missingContainer);++ try (DockerClient docker = dockerClient(transport)) {+ RuntimeException thrown = assertThrows(RuntimeException.class,+ () -> RetryingTarCopier.copyHostResource(docker, "c-1", "/var/lib/rancher",+ file.toString(), MAX_ATTEMPTS, 0L));++ assertInstanceOf(NotFoundException.class, thrown.getCause());+ }++ verify(transport, times(1)).execute(any());}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/UserDataPipelineTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/UserDataPipelineTest.javaindex f64b45b31b..f1c7e98e78 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/UserDataPipelineTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/UserDataPipelineTest.java@@ -1,21 +1,111 @@package io.github.hectorvent.floci.core.common.docker;+import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;+import com.github.dockerjava.api.exception.NotFoundException;+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;+import java.io.ByteArrayInputStream;import java.io.ByteArrayOutputStream;import java.io.IOException;+import java.io.InputStream;import java.nio.charset.StandardCharsets;+import java.time.Duration;+import java.util.ArrayList;import java.util.Base64;import java.util.List;+import java.util.concurrent.atomic.AtomicInteger;import java.util.zip.GZIPOutputStream;+import static org.junit.jupiter.api.Assertions.assertArrayEquals;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNotSame;+import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.RETURNS_SELF;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;class UserDataPipelineTest {+ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void retriesConsumedScriptUploadBeforeExecutingExactlyOnce(boolean consumeCompletely) {+ String script = "#!/bin/sh\nprintf 'bootstrap complete\\n'\n";+ DockerClient docker = mock(DockerClient.class);+ CopyArchiveToContainerCmd copyCmd = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);+ when(docker.copyArchiveToContainerCmd("container-id")).thenReturn(copyCmd);+ List<InputStream> streams = new ArrayList<>();+ when(copyCmd.withTarInputStream(any(InputStream.class))).thenAnswer(invocation -> {+ streams.add(invocation.getArgument(0));+ return copyCmd;+ });+ AtomicInteger attempts = new AtomicInteger();+ when(copyCmd.exec()).thenAnswer(invocation -> {+ int attempt = attempts.incrementAndGet();+ InputStream stream = streams.getLast();+ if (attempt == 1 && !consumeCompletely) {+ assertEquals(513, stream.readNBytes(513).length, "consume the tar header and part of the script");+ } else {+ byte[] archive = stream.readAllBytes();+ try (TarArchiveInputStream tar = new TarArchiveInputStream(new ByteArrayInputStream(archive))) {+ TarArchiveEntry entry = tar.getNextEntry();+ assertNotNull(entry, "every attempt must begin with a complete tar header");+ assertEquals("user-data.sh", entry.getName());+ assertEquals(0755, entry.getMode() & 0777);+ assertArrayEquals(script.getBytes(StandardCharsets.UTF_8), tar.readAllBytes());+ assertNull(tar.getNextEntry(), "the archive holds only the user-data script");+ }+ }+ if (attempt == 1) {+ throw new RuntimeException(new IOException("Connection reset"));+ }+ return null;+ });+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(docker, "container-id", 0L,+ "bootstrap complete\n", "");+ List<String> output = new ArrayList<>();++ UserDataPipeline.ExecutionResult result = UserDataPipeline.executeUserData(docker, "container-id",+ "test instance", script, Duration.ofSeconds(1), output::add, null, 0L);++ assertTrue(result.isSuccess(), result.getFailureMessage());+ assertEquals(2, attempts.get());+ assertEquals(2, streams.size());+ assertNotSame(streams.getFirst(), streams.get(1), "a consumed tar stream cannot be replayed");+ assertEquals(List.of(List.of(UserDataPipeline.USER_DATA_SCRIPT_PATH)), commands);+ assertEquals(List.of("bootstrap complete"), output);+ verify(copyCmd, times(2)).withRemotePath("/var/lib");+ verify(docker).execStartCmd("exec-container-id");+ }++ @Test+ void daemonRejectionDoesNotRetryScriptUploadOrExecuteTheScript() {+ DockerClient docker = mock(DockerClient.class);+ CopyArchiveToContainerCmd copyCmd = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);+ when(docker.copyArchiveToContainerCmd("container-id")).thenReturn(copyCmd);+ when(copyCmd.exec()).thenThrow(new NotFoundException("Could not find the file /var/lib in container"));++ UserDataPipeline.ExecutionResult result = UserDataPipeline.executeUserData(docker, "container-id",+ "test instance", "#!/bin/sh\nexit 0\n", Duration.ofSeconds(1), null, null);++ assertFalse(result.isSuccess());+ assertTrue(result.getFailureMessage().contains("Could not find the file /var/lib in container"));+ verify(copyCmd).exec();+ verify(docker, never()).execCreateCmd(anyString());+ }+@Testvoid extractsPlainShellScript() {String script = "#!/bin/bash\necho 'hello world'\n";diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerArchiveRetryTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerArchiveRetryTest.javanew file mode 100644index 0000000000..5683a44d2c--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerArchiveRetryTest.java@@ -0,0 +1,346 @@+package io.github.hectorvent.floci.services.eks;++import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;+import com.github.dockerjava.api.exception.NotFoundException;+import com.github.dockerjava.core.command.CopyArchiveToContainerCmdImpl;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.config.FlociCertificateAuthority;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;+import io.github.hectorvent.floci.core.common.docker.PortAllocator;+import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;+import io.github.hectorvent.floci.services.eks.model.Cluster;+import io.github.hectorvent.floci.services.eks.model.ClusterIdentity;+import io.github.hectorvent.floci.services.eks.model.ClusterOidcKey;+import io.github.hectorvent.floci.services.eks.model.LogSetup;+import io.github.hectorvent.floci.services.eks.model.Logging;+import io.github.hectorvent.floci.services.eks.model.OidcIdentity;+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.nio.file.attribute.PosixFilePermissions;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.Optional;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.function.Function;+import java.util.zip.GZIPInputStream;++import static io.github.hectorvent.floci.testing.InlineMocks.inlineMock;+import static org.junit.jupiter.api.Assertions.assertArrayEquals;+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.RETURNS_SELF;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++class EksClusterManagerArchiveRetryTest {++ private static final String CONTAINER_ID = "container-1";+ private static final String CA_PEM = "-----BEGIN CERTIFICATE-----\ntest-only-ca\n-----END CERTIFICATE-----\n";+ private static final byte[] PRIVATE_KEY = "test-only generated private key\n".getBytes(StandardCharsets.UTF_8);+ private static final byte[] PUBLIC_KEY = "test-only generated public key\n".getBytes(StandardCharsets.UTF_8);++ @TempDir+ Path tempDir;++ private final List<ArchiveUpload> uploads = new ArrayList<>();+ private Function<ArchiveUpload, RuntimeException> failure = upload -> null;+ private EmulatorConfig.EksServiceConfig eks;+ private ContainerLifecycleManager lifecycleManager;+ private EksOidcService oidcService;+ private EksClusterManager manager;++ @BeforeEach+ void setUp() {+ EmulatorConfig config = mock(EmulatorConfig.class);+ EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);+ eks = mock(EmulatorConfig.EksServiceConfig.class);+ EmulatorConfig.TlsConfig tls = mock(EmulatorConfig.TlsConfig.class);+ when(config.services()).thenReturn(services);+ when(services.eks()).thenReturn(eks);+ when(config.tls()).thenReturn(tls);+ when(config.port()).thenReturn(4566);+ when(config.defaultAccountId()).thenReturn("000000000000");+ when(tls.enabled()).thenReturn(true);+ when(eks.podIdentityWebhook()).thenReturn(true);+ when(eks.dataPath()).thenReturn(tempDir.toString());+ when(eks.defaultImage()).thenReturn("rancher/k3s:v1.30.0-k3s1");+ when(eks.imageTemplate()).thenReturn(Optional.empty());+ when(eks.dockerNetwork()).thenReturn(Optional.empty());+ when(eks.endpointMode()).thenReturn("host");++ DockerClient docker = mock(DockerClient.class);+ CopyArchiveToContainerCmd.Exec copyExec = mock(CopyArchiveToContainerCmd.Exec.class);+ when(copyExec.exec(any())).thenAnswer(invocation -> {+ CopyArchiveToContainerCmd command = invocation.getArgument(0);+ ArchiveUpload upload = consumeArchive(command);+ uploads.add(upload);+ RuntimeException copyFailure = failure.apply(upload);+ if (copyFailure != null) {+ throw copyFailure;+ }+ return null;+ });+ when(docker.copyArchiveToContainerCmd(anyString())).thenAnswer(invocation ->+ new CopyArchiveToContainerCmdImpl(copyExec, invocation.getArgument(0)));+ lifecycleManager = mock(ContainerLifecycleManager.class);+ when(lifecycleManager.getDockerClient()).thenReturn(docker);+ when(lifecycleManager.create(any())).thenReturn(CONTAINER_ID);+ when(lifecycleManager.startCreated(any(), any())).thenReturn(new ContainerInfo(CONTAINER_ID, Map.of()));++ ContainerBuilder containerBuilder = mock(ContainerBuilder.class);+ ContainerBuilder.Builder builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);+ when(containerBuilder.newContainer(anyString())).thenReturn(builder);+ when(builder.build()).thenReturn(new ContainerSpec("test-image"));+ DockerHostResolver dockerHostResolver = mock(DockerHostResolver.class);+ when(dockerHostResolver.resolve()).thenReturn("host.docker.internal");+ RegionResolver regionResolver = mock(RegionResolver.class);+ when(regionResolver.getAccountId()).thenReturn("000000000000");+ when(regionResolver.getDefaultRegion()).thenReturn("us-east-1");+ FlociCertificateAuthority certificateAuthority = inlineMock(FlociCertificateAuthority.class);+ when(certificateAuthority.caPem()).thenReturn(CA_PEM);+ oidcService = mock(EksOidcService.class);++ manager = new EksClusterManager(containerBuilder, lifecycleManager,+ mock(ContainerDetector.class), mock(PortAllocator.class), dockerHostResolver,+ mock(EcrRegistryManager.class), config, regionResolver, null, oidcService, certificateAuthority,+ null, null, 0L);+ }++ @Test+ void auditPolicyRetriesAfterTheFirstArchiveIsConsumed() throws IOException {+ Path policy = tempDir.resolve("audit-policy.yaml");+ byte[] content = EksClusterManager.buildAuditPolicy().getBytes(StandardCharsets.UTF_8);+ Files.write(policy, content);+ failFirstUploadOf("audit-policy.yaml");++ manager.copyAuditPolicyIntoContainer(CONTAINER_ID, policy.toString(), "demo");++ assertEquals(2, uploads.size(), "a consumed audit policy archive must be rebuilt after a broken pipe");+ for (ArchiveUpload upload : uploads) {+ assertHostFileUpload(upload, policy, content);+ }+ }++ @Test+ void signingPrivateKeyRetriesBeforeUploadingThePublicKey() throws IOException {+ EksClusterManager.SigningKeyFiles keys = signingKeyFiles();+ failFirstUploadOf("sa-signing-key.pem");++ manager.copySigningKeysIntoContainer(CONTAINER_ID, keys, "demo");++ assertEquals(3, uploads.size(), "the private key must recover before the public key is uploaded");+ assertHostFileUpload(uploads.get(0), keys.signingKeyPath(), PRIVATE_KEY);+ assertHostFileUpload(uploads.get(1), keys.signingKeyPath(), PRIVATE_KEY);+ assertHostFileUpload(uploads.get(2), keys.publicKeyPath(), PUBLIC_KEY);+ assertSigningKeySourcesRemainPrivate(keys);+ }++ @Test+ void signingPublicKeyRetriesWithoutUploadingTheSuccessfulPrivateKeyAgain() throws IOException {+ EksClusterManager.SigningKeyFiles keys = signingKeyFiles();+ failFirstUploadOf("sa-public-key.pem");++ manager.copySigningKeysIntoContainer(CONTAINER_ID, keys, "demo");++ assertEquals(3, uploads.size(), "only the failed public key upload must be retried");+ assertHostFileUpload(uploads.get(0), keys.signingKeyPath(), PRIVATE_KEY);+ assertHostFileUpload(uploads.get(1), keys.publicKeyPath(), PUBLIC_KEY);+ assertHostFileUpload(uploads.get(2), keys.publicKeyPath(), PUBLIC_KEY);+ assertSigningKeySourcesRemainPrivate(keys);+ }++ @Test+ void dockerJavaDirectHostResourceCopyUses0644ForPrivateSourceFiles() throws IOException {+ EksClusterManager.SigningKeyFiles keys = signingKeyFiles();++ lifecycleManager.getDockerClient().copyArchiveToContainerCmd(CONTAINER_ID)+ .withHostResource(keys.signingKeyPath().toString())+ .withRemotePath("/etc")+ .exec();++ assertEquals(1, uploads.size());+ assertHostFileUpload(uploads.getFirst(), keys.signingKeyPath(), PRIVATE_KEY);+ assertSigningKeySourcesRemainPrivate(keys);+ }++ @Test+ void containerdLinkRetriesWithEveryDirectoryAndTheSymlinkIntact() {+ failFirstUploadOf("etc/containerd/certs.d");++ manager.linkContainerdCertsDir(CONTAINER_ID, "demo");++ assertEquals(2, uploads.size(), "the entire multi-entry archive must survive the retry");+ for (ArchiveUpload upload : uploads) {+ assertEquals(CONTAINER_ID, upload.containerId());+ assertEquals("/", upload.remotePath());+ assertEquals(3, upload.entries().size());+ assertDirectory(upload.entries().get(0), "var/lib/rancher/k3s/agent/etc/containerd/certs.d/");+ assertDirectory(upload.entries().get(1), "etc/containerd/");+ ArchiveEntry link = upload.entries().get(2);+ assertEquals("etc/containerd/certs.d", link.name());+ assertTrue(link.symbolicLink());+ assertFalse(link.directory());+ assertEquals(0777, link.mode());+ assertEquals("/var/lib/rancher/k3s/agent/etc/containerd/certs.d", link.linkName());+ assertArrayEquals(new byte[0], link.content());+ }+ }++ @Test+ void podIdentityManifestRetriesWithItsFullContentAndMode() {+ failFirstUploadOf("server/manifests/floci-eks-pod-identity.yaml");+ byte[] expected = EksClusterManager.buildPodIdentityWebhookConfiguration(+ "https://host.docker.internal:4566/_floci/eks/clusters/demo/pod-identity-webhook/scope/000000000000",+ CA_PEM).getBytes(StandardCharsets.UTF_8);++ manager.registerPodIdentityWebhook(CONTAINER_ID, cluster());++ assertEquals(2, uploads.size(), "the pod identity manifest must be rebuilt after the consumed upload fails");+ for (ArchiveUpload upload : uploads) {+ assertFileUpload(upload, "/var/lib/rancher/k3s", "server/manifests/floci-eks-pod-identity.yaml", expected);+ }+ }++ @Test+ void publicKeyDaemonRejectionDoesNotRetryEitherKeyOrPropagate() throws IOException {+ EksClusterManager.SigningKeyFiles keys = signingKeyFiles();+ failure = upload -> containsEntry(upload, "sa-public-key.pem")+ ? new NotFoundException("No such container: " + CONTAINER_ID) : null;++ assertDoesNotThrow(() -> manager.copySigningKeysIntoContainer(CONTAINER_ID, keys, "demo"));++ assertEquals(2, uploads.size());+ assertHostFileUpload(uploads.get(0), keys.signingKeyPath(), PRIVATE_KEY);+ assertHostFileUpload(uploads.get(1), keys.publicKeyPath(), PUBLIC_KEY);+ assertSigningKeySourcesRemainPrivate(keys);+ }++ @Test+ void archiveDaemonRejectionsAreNotRetriedAndLeaveTheClusterRunning() {+ failure = upload -> new NotFoundException("No such container: " + CONTAINER_ID);+ when(eks.irsaSigningKey()).thenReturn(true);+ Cluster cluster = cluster();+ cluster.setLogging(new Logging(List.of(new LogSetup(List.of("audit"), true))));+ ClusterOidcKey key = new ClusterOidcKey("https://test.invalid/issuer", "fixture", "public", "private");+ when(oidcService.ensureKeyForAccount("000000000000", "demo", key.getIssuer())).thenReturn(key);+ when(oidcService.exportSigningKeyPem(key)).thenReturn(new String(PRIVATE_KEY, StandardCharsets.UTF_8));+ when(oidcService.exportPublicKeyPem(key)).thenReturn(new String(PUBLIC_KEY, StandardCharsets.UTF_8));++ assertDoesNotThrow(() -> manager.startCluster(cluster));++ assertEquals(List.of("audit-policy.yaml", "etc/containerd/certs.d",+ "server/manifests/floci-eks-pod-identity.yaml", "sa-signing-key.pem"),+ uploads.stream().map(upload -> upload.entries().getLast().name()).toList(),+ "each rejected feature is attempted once and startup continues with reduced functionality");+ verify(lifecycleManager).startCreated(any(), any());+ assertEquals(CONTAINER_ID, cluster.getContainerId());+ }++ private EksClusterManager.SigningKeyFiles signingKeyFiles() throws IOException {+ Path signingKey = tempDir.resolve("sa-signing-key.pem");+ Path publicKey = tempDir.resolve("sa-public-key.pem");+ Files.write(signingKey, PRIVATE_KEY);+ Files.write(publicKey, PUBLIC_KEY);+ if (Files.getFileStore(tempDir).supportsFileAttributeView("posix")) {+ Files.setPosixFilePermissions(signingKey, PosixFilePermissions.fromString("rw-------"));+ Files.setPosixFilePermissions(publicKey, PosixFilePermissions.fromString("rw-------"));+ }+ return new EksClusterManager.SigningKeyFiles(signingKey, publicKey);+ }++ private static void assertSigningKeySourcesRemainPrivate(EksClusterManager.SigningKeyFiles keys) throws IOException {+ if (Files.getFileStore(keys.signingKeyPath()).supportsFileAttributeView("posix")) {+ assertEquals(PosixFilePermissions.fromString("rw-------"), Files.getPosixFilePermissions(keys.signingKeyPath()));+ assertEquals(PosixFilePermissions.fromString("rw-------"), Files.getPosixFilePermissions(keys.publicKeyPath()));+ }+ }++ private static Cluster cluster() {+ Cluster cluster = new Cluster();+ cluster.setName("demo");+ cluster.setAccountId("000000000000");+ cluster.setIdentity(new ClusterIdentity(new OidcIdentity("https://test.invalid/issuer")));+ return cluster;+ }++ private void failFirstUploadOf(String name) {+ AtomicInteger attempts = new AtomicInteger();+ failure = upload -> containsEntry(upload, name) && attempts.incrementAndGet() == 1+ ? new RuntimeException(new IOException("Broken pipe")) : null;+ }++ private static boolean containsEntry(ArchiveUpload upload, String name) {+ return upload.entries().stream().anyMatch(entry -> entry.name().equals(name));+ }++ private static ArchiveUpload consumeArchive(CopyArchiveToContainerCmd command) throws IOException {+ InputStream body = command.getTarInputStream();+ if (command.getHostResource() != null) {+ body = new GZIPInputStream(body);+ }+ List<ArchiveEntry> entries = new ArrayList<>();+ try (TarArchiveInputStream tar = new TarArchiveInputStream(body)) {+ TarArchiveEntry entry;+ while ((entry = tar.getNextEntry()) != null) {+ entries.add(new ArchiveEntry(entry.getName(), entry.getMode() & 07777,+ entry.isDirectory(), entry.isSymbolicLink(), entry.getLinkName(), tar.readAllBytes()));+ }+ body.transferTo(OutputStream.nullOutputStream());+ }+ return new ArchiveUpload(command.getContainerId(), command.getRemotePath(), command.getHostResource(), entries);+ }++ private static void assertHostFileUpload(ArchiveUpload upload, Path source, byte[] content) {+ assertEquals(source.toString(), upload.hostResource());+ // docker-java's File-based tar entries default to 0644 even when the source file is protected with 0600.+ assertFileUpload(upload, "/etc", source.getFileName().toString(), content);+ }++ private static void assertFileUpload(ArchiveUpload upload, String remotePath, String name, byte[] content) {+ assertEquals(CONTAINER_ID, upload.containerId());+ assertEquals(remotePath, upload.remotePath());+ assertEquals(1, upload.entries().size());+ ArchiveEntry entry = upload.entries().getFirst();+ assertEquals(name, entry.name());+ assertEquals(0644, entry.mode());+ assertFalse(entry.directory());+ assertFalse(entry.symbolicLink());+ assertArrayEquals(content, entry.content());+ }++ private static void assertDirectory(ArchiveEntry entry, String name) {+ assertEquals(name, entry.name());+ assertTrue(entry.directory());+ assertFalse(entry.symbolicLink());+ assertEquals(0755, entry.mode());+ assertArrayEquals(new byte[0], entry.content());+ }++ private record ArchiveUpload(String containerId, String remotePath, String hostResource, List<ArchiveEntry> entries) {}++ private record ArchiveEntry(String name, int mode, boolean directory, boolean symbolicLink,+ String linkName, byte[] content) {}+}diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.javaindex cb6e3e796f..0590e6dce1 100644--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java@@ -62,6 +62,8 @@ import org.junit.jupiter.params.provider.ValueSource;import org.mockito.ArgumentCaptor;import org.mockito.Mockito;+import java.io.ByteArrayInputStream;+import java.io.ByteArrayOutputStream;import java.io.Closeable;import java.io.InputStream;import java.nio.charset.StandardCharsets;@@ -1330,15 +1332,23 @@ class EksClusterManagerTest {@Testvoid copiesArchiveWithTargetDirectoryAndSymlinkAtContainerRoot() throws Exception {+ ArgumentCaptor<InputStream> archive = ArgumentCaptor.forClass(InputStream.class);+ when(copyCmd.withTarInputStream(archive.capture())).thenReturn(copyCmd);+ ByteArrayOutputStream archiveBytes = new ByteArrayOutputStream();+ when(copyCmd.exec()).thenAnswer(invocation -> {+ archive.getValue().transferTo(archiveBytes);+ return null;+ });+manager.linkContainerdCertsDir("container-1", "demo");verify(dockerClient).copyArchiveToContainerCmd("container-1");verify(copyCmd).withRemotePath("/");verify(copyCmd).exec();- ArgumentCaptor<InputStream> archive = ArgumentCaptor.forClass(InputStream.class);- verify(copyCmd).withTarInputStream(archive.capture());- try (TarArchiveInputStream tar = new TarArchiveInputStream(archive.getValue())) {+ verify(copyCmd).withTarInputStream(any());+ try (TarArchiveInputStream tar = new TarArchiveInputStream(+ new ByteArrayInputStream(archiveBytes.toByteArray()))) {TarArchiveEntry targetDir = tar.getNextEntry();assertNotNull(targetDir);assertEquals("var/lib/rancher/k3s/agent/etc/containerd/certs.d/", targetDir.getName());diff --git a/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2IntegrationTest.javaindex 3e4570e571..dc2c446d99 100644--- a/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2IntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2IntegrationTest.java@@ -481,27 +481,33 @@ class ResourceExplorer2IntegrationTest {}@Test- void searchWithEmptyQueryReturnsAllResources() {- Response response = given()+ void searchWithEmptyQueryReturnsEveryResourceInItsView() {+ String bucketArn = Arn.global(AwsRegions.partitionFor("us-east-1"), "s3", "", "re2-test-bucket")+ .toString();+ String viewArn = trackView(given().header("Authorization", AUTH).contentType("application/json")- .body("""- {"QueryString": ""}- """)+ .body(Map.of("ViewName", "empty-query-fixture-view",+ "Filters", Map.of("FilterString", "id:" + bucketArn)))+ .when()+ .post("/CreateView")+ .then()+ .statusCode(200)+ .extract().path("View.ViewArn"));++ given()+ .header("Authorization", AUTH)+ .contentType("application/json")+ .body(Map.of("QueryString", "", "ViewArn", viewArn)).when().post("/Search").then().statusCode(200)- .body("Count.TotalResources", greaterThanOrEqualTo(0))- .body("Resources", notNullValue())- .extract().response();-- // Unscoped, so how many resources the shared emulator holds is not knowable here.- // Complete says whether the 1000-result cap bit, which is exactly what TotalResources- // reaching the cap means — assert that relationship rather than a fixed value.- int total = response.path("Count.TotalResources");- boolean complete = response.path("Count.Complete");- assertEquals(total < 1000, complete);+ .body("Count.TotalResources", equalTo(1))+ .body("Count.Complete", equalTo(true))+ .body("Resources.size()", equalTo(1))+ .body("Resources[0].Arn", equalTo(bucketArn))+ .body("ViewArn", equalTo(viewArn));}}diff --git a/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2ListVsSearchCapTest.java b/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2ListVsSearchCapTest.javaindex fb17f2b89e..8c40fd12ac 100644--- a/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2ListVsSearchCapTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2ListVsSearchCapTest.java@@ -14,6 +14,8 @@ import io.github.hectorvent.floci.services.resourceexplorer2.model.View;import io.quarkus.runtime.StartupEvent;import jakarta.enterprise.inject.Instance;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;import java.time.Instant;import java.util.ArrayList;@@ -134,6 +136,17 @@ class ResourceExplorer2ListVsSearchCapTest {assertEquals(1000, lastTotal);}+ @ParameterizedTest+ @CsvSource({"999,999,true", "1000,1000,true", "1001,1000,false"})+ void searchCountDistinguishesExactCapFromTruncation(int resourceCount, int expectedTotal, boolean complete) {+ ObjectNode result = newServiceWith(resourceCount).search("", 1000, null, null, REGION);++ assertEquals(expectedTotal, result.path("Resources").size());+ assertEquals(expectedTotal, result.path("Count").path("TotalResources").asInt());+ assertEquals(complete, result.path("Count").path("Complete").asBoolean());+ assertFalse(result.has("NextToken"));+ }+@Testvoid oneThrowingProviderIsSkippedAndHealthyProvidersStillSurface() {// Discovery aggregates every provider. A single provider blowing up (bad stored data, a bug)