Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(docker): retry core and EKS archive uploads (#5916)

Aniket Shuklad951fd532cd5
diff --git a/docs/configuration/docker.md b/docs/configuration/docker.md
index f70f50e9ef..72e1d2195d 100644
--- a/docs/configuration/docker.md
+++ b/docs/configuration/docker.md
@@ -322,6 +322,13 @@ All of Floci's short-lived docker calls (create/start/inspect/remove container,
A request is only replayed when doing so cannot change semantics: requests carrying a one-shot upload stream (e.g. copying an archive into a container), bidirectional attach streams, and `exec` requests (which would re-run the command) are never retried. Log-follow connections use a separate transport that does not retry at all. Bodyless actions whose replay changes the outcome (unnamed container create, network create, container restart, network connect and disconnect) are not retried either. A tar-writer failure on the Floci host (for example an unreadable source file) is reported at once instead of retried.
+Archive uploads that use the shared tar-copy helper retry above the transport layer, rebuilding
+the upload stream for each attempt with the same six-attempt backoff. This includes user-data
+scripts, the container CA bundle, and EKS audit policies, IRSA signing keys, the containerd
+certificate-directory link and the Pod Identity manifest. Only the upload is retried, not
+script execution. Daemon rejections still stop immediately, and each caller keeps its existing
+failure handling after retries are exhausted.
+
## Full Reference
| Environment variable | Default | Description |
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index 4fee6050d6..ca4b409d2d 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -30,12 +30,8 @@ import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.Lau
import io.quarkus.runtime.annotations.RegisterForReflection;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.jboss.logging.Logger;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -91,6 +87,7 @@ public class ContainerLifecycleManager {
private final ImageCacheService imageCacheService;
private final ContainerDetector containerDetector;
private final EmulatorConfig config;
+ private final long archiveRetryBackoffMillis;
/** Volumes whose shared-ownership root has already been initialised this process (run-once guard). */
private final ConcurrentHashMap<String, Boolean> initializedSharedVolumes = new ConcurrentHashMap<>();
@@ -103,10 +100,20 @@ public class ContainerLifecycleManager {
ImageCacheService imageCacheService,
ContainerDetector containerDetector,
EmulatorConfig config) {
+ this(dockerClient, imageCacheService, containerDetector, config,
+ RetryingTarCopier.DEFAULT_BACKOFF_MS);
+ }
+
+ ContainerLifecycleManager(DockerClient dockerClient,
+ ImageCacheService imageCacheService,
+ ContainerDetector containerDetector,
+ EmulatorConfig config,
+ long archiveRetryBackoffMillis) {
this.dockerClient = dockerClient;
this.imageCacheService = imageCacheService;
this.containerDetector = containerDetector;
this.config = config;
+ this.archiveRetryBackoffMillis = archiveRetryBackoffMillis;
}
/**
@@ -331,18 +338,9 @@ public class ContainerLifecycleManager {
private boolean copyCaBundle(String containerId, Path bundle) {
try {
byte[] content = Files.readAllBytes(bundle);
- ByteArrayOutputStream archive = new ByteArrayOutputStream(content.length + 1024);
- try (TarArchiveOutputStream tar = new TarArchiveOutputStream(archive)) {
- TarArchiveEntry entry = new TarArchiveEntry(ContainerCaBundle.FILE_NAME);
- entry.setSize(content.length);
- tar.putArchiveEntry(entry);
- tar.write(content);
- tar.closeArchiveEntry();
- }
- dockerClient.copyArchiveToContainerCmd(containerId)
- .withRemotePath(ContainerCaBundle.CONTAINER_DIR)
- .withTarInputStream(new ByteArrayInputStream(archive.toByteArray()))
- .exec();
+ RetryingTarCopier.copyBytes(dockerClient, containerId, ContainerCaBundle.CONTAINER_DIR,
+ ContainerCaBundle.FILE_NAME, content, 0644,
+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);
LOG.debugv("Copied the CA bundle into container {0} at {1}", containerId, ContainerCaBundle.CONTAINER_PATH);
return true;
} catch (Exception e) {
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.java
index 1f44874586..aec79c55ad 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.core.common.docker;
import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.exception.DockerClientException;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.jboss.logging.Logger;
@@ -30,8 +31,8 @@ public final class RetryingTarCopier {
private static final Logger LOG = Logger.getLogger(RetryingTarCopier.class);
- static final int DEFAULT_MAX_ATTEMPTS = 6;
- static final long DEFAULT_BACKOFF_MS = 500L;
+ public static final int DEFAULT_MAX_ATTEMPTS = 6;
+ public static final long DEFAULT_BACKOFF_MS = 500L;
/**
* Buffer for the tar-streaming pipe. The default {@link PipedInputStream} buffer is only 1KB,
@@ -154,11 +155,22 @@ public final class RetryingTarCopier {
public static void copyHostResource(DockerClient docker, String containerId, String remotePath,
String hostPath, int maxAttempts, long backoffMillis) {
- retry(hostPath, containerId, maxAttempts, backoffMillis, () ->
+ retry(hostPath, containerId, maxAttempts, backoffMillis, () -> {
+ try {
docker.copyArchiveToContainerCmd(containerId)
.withHostResource(hostPath)
.withRemotePath(remotePath)
- .exec());
+ .exec();
+ } catch (DockerClientException e) {
+ // These docker-java wrappers identify local archive preparation and temporary-file I/O.
+ String message = e.getMessage();
+ if (message != null && (message.startsWith("Unable to perform tar on host resource ")
+ || message.startsWith("Unable to read temp file "))) {
+ throw new DockerRetry.LocalFailure(message, e);
+ }
+ throw e;
+ }
+ });
}
/** Single-entry tar bytes, reusable across attempts and callers. */
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/UserDataPipeline.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/UserDataPipeline.java
index ba18990805..136cf83f3b 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/UserDataPipeline.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/UserDataPipeline.java
@@ -3,8 +3,6 @@ package io.github.hectorvent.floci.core.common.docker;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.async.ResultCallback;
import com.github.dockerjava.api.model.Frame;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.jboss.logging.Logger;
import java.io.ByteArrayInputStream;
@@ -203,20 +201,6 @@ public final class UserDataPipeline {
return false;
}
- public static byte[] buildSingleFileTar(String filename, byte[] content, int mode) throws IOException {
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- try (TarArchiveOutputStream tar = new TarArchiveOutputStream(bos)) {
- tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
- TarArchiveEntry entry = new TarArchiveEntry(filename);
- entry.setSize(content.length);
- entry.setMode(mode);
- tar.putArchiveEntry(entry);
- tar.write(content);
- tar.closeArchiveEntry();
- }
- return bos.toByteArray();
- }
-
public static String summarizeUserDataOutput(BoundedOutput output) {
if (output == null) {
return "(no output)";
@@ -239,6 +223,20 @@ public final class UserDataPipeline {
Duration timeout,
Consumer<String> outputLineConsumer,
Set<ResultCallback<Frame>> activeCallbacks
+ ) {
+ return executeUserData(dockerClient, containerId, contextDescription, userData, timeout,
+ outputLineConsumer, activeCallbacks, RetryingTarCopier.DEFAULT_BACKOFF_MS);
+ }
+
+ static ExecutionResult executeUserData(
+ DockerClient dockerClient,
+ String containerId,
+ String contextDescription,
+ String userData,
+ Duration timeout,
+ Consumer<String> outputLineConsumer,
+ Set<ResultCallback<Frame>> activeCallbacks,
+ long archiveRetryBackoffMillis
) {
if (dockerClient == null || containerId == null || containerId.isBlank()) {
return ExecutionResult.skipped("No container available");
@@ -255,7 +253,7 @@ public final class UserDataPipeline {
String scriptContent = shellScripts.get(i);
ExecutionResult partResult = executeSingleScript(
dockerClient, containerId, contextDescription, scriptContent, partNumber, partCount,
- effectiveTimeout, outputLineConsumer, activeCallbacks);
+ effectiveTimeout, outputLineConsumer, activeCallbacks, archiveRetryBackoffMillis);
if (!partResult.isSuccess()) {
return partResult;
}
@@ -272,15 +270,13 @@ public final class UserDataPipeline {
int partCount,
Duration timeout,
Consumer<String> outputLineConsumer,
- Set<ResultCallback<Frame>> activeCallbacks
+ Set<ResultCallback<Frame>> activeCallbacks,
+ long archiveRetryBackoffMillis
) {
byte[] script = scriptContent.getBytes(StandardCharsets.UTF_8);
try {
- byte[] tar = buildSingleFileTar("user-data.sh", script, 0755);
- dockerClient.copyArchiveToContainerCmd(containerId)
- .withRemotePath("/var/lib")
- .withTarInputStream(new ByteArrayInputStream(tar))
- .exec();
+ RetryingTarCopier.copyBytes(dockerClient, containerId, "/var/lib", "user-data.sh", script, 0755,
+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);
String execId = dockerClient.execCreateCmd(containerId)
.withCmd(USER_DATA_SCRIPT_PATH)
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
index 4b25652e0d..5de58680a1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
@@ -62,7 +62,6 @@ import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.jboss.logging.Logger;
-import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.File;
@@ -158,6 +157,7 @@ public class EksClusterManager
private final ContainerBuilder containerBuilder;
private final ContainerLifecycleManager lifecycleManager;
+ private final long archiveRetryBackoffMillis;
private final ContainerDetector containerDetector;
private final PortAllocator portAllocator;
private final DockerHostResolver dockerHostResolver;
@@ -424,6 +424,26 @@ public class EksClusterManager
FlociCertificateAuthority certificateAuthority,
ContainerLogStreamer logStreamer,
StorageBackend<String, Nodegroup> nodeGroupStorage) {
+ this(containerBuilder, lifecycleManager, containerDetector, portAllocator,
+ dockerHostResolver, ecrRegistryManager, config, regionResolver, metadataServer,
+ oidcService, certificateAuthority, logStreamer, nodeGroupStorage,
+ RetryingTarCopier.DEFAULT_BACKOFF_MS);
+ }
+
+ EksClusterManager(ContainerBuilder containerBuilder,
+ ContainerLifecycleManager lifecycleManager,
+ ContainerDetector containerDetector,
+ PortAllocator portAllocator,
+ DockerHostResolver dockerHostResolver,
+ EcrRegistryManager ecrRegistryManager,
+ EmulatorConfig config,
+ RegionResolver regionResolver,
+ Ec2MetadataServer metadataServer,
+ EksOidcService oidcService,
+ FlociCertificateAuthority certificateAuthority,
+ ContainerLogStreamer logStreamer,
+ StorageBackend<String, Nodegroup> nodeGroupStorage,
+ long archiveRetryBackoffMillis) {
this.containerBuilder = containerBuilder;
this.lifecycleManager = lifecycleManager;
this.containerDetector = containerDetector;
@@ -437,6 +457,7 @@ public class EksClusterManager
this.certificateAuthority = certificateAuthority;
this.logStreamer = logStreamer;
this.nodeGroupStorage = nodeGroupStorage;
+ this.archiveRetryBackoffMillis = archiveRetryBackoffMillis;
}
@Inject
@@ -1554,7 +1575,8 @@ public class EksClusterManager
private void copyWebhookIntoContainer(String containerId, String localFile, String clusterName) {
try {
RetryingTarCopier.copyHostResource(lifecycleManager.getDockerClient(), containerId,
- WEBHOOK_CONFIG_DIR, localFile);
+ WEBHOOK_CONFIG_DIR, localFile,
+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);
} catch (Exception e) {
LOG.warnv("EKS token-webhook may not authenticate for cluster {0}: could not copy kubeconfig "
+ "into the k3s container: {1}", clusterName, e.getMessage());
@@ -1592,10 +1614,8 @@ public class EksClusterManager
try {
DockerClient dockerClient = lifecycleManager.getDockerClient();
if (dockerClient != null) {
- dockerClient.copyArchiveToContainerCmd(containerId)
- .withHostResource(localFile)
- .withRemotePath(AUDIT_POLICY_DIR)
- .exec();
+ RetryingTarCopier.copyHostResource(dockerClient, containerId, AUDIT_POLICY_DIR, localFile,
+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);
LOG.debugv("Injected audit policy file into k3s container {0} for cluster {1}",
containerId, clusterName);
}
@@ -1919,16 +1939,12 @@ public class EksClusterManager
*/
void copySigningKeysIntoContainer(String containerId, SigningKeyFiles keyFiles, String clusterName) {
try {
- lifecycleManager.getDockerClient()
- .copyArchiveToContainerCmd(containerId)
- .withHostResource(keyFiles.signingKeyPath().toString())
- .withRemotePath(WEBHOOK_CONFIG_DIR)
- .exec();
- lifecycleManager.getDockerClient()
- .copyArchiveToContainerCmd(containerId)
- .withHostResource(keyFiles.publicKeyPath().toString())
- .withRemotePath(WEBHOOK_CONFIG_DIR)
- .exec();
+ RetryingTarCopier.copyHostResource(lifecycleManager.getDockerClient(), containerId,
+ WEBHOOK_CONFIG_DIR, keyFiles.signingKeyPath().toString(),
+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);
+ RetryingTarCopier.copyHostResource(lifecycleManager.getDockerClient(), containerId,
+ WEBHOOK_CONFIG_DIR, keyFiles.publicKeyPath().toString(),
+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);
LOG.debugv("Injected IRSA OIDC signing keypair into k3s container {0} for cluster {1}",
containerId, clusterName);
} catch (Exception e) {
@@ -1983,7 +1999,8 @@ public class EksClusterManager
"registries.yaml"), content, clusterName);
try {
RetryingTarCopier.copyBytes(lifecycleManager.getDockerClient(), containerId, "/etc",
- REGISTRIES_TAR_ENTRY, content.getBytes(StandardCharsets.UTF_8), 0644);
+ REGISTRIES_TAR_ENTRY, content.getBytes(StandardCharsets.UTF_8), 0644,
+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);
LOG.infov("Injected ECR registry mirror ({0}) into k3s cluster {1}", endpoint, clusterName);
} catch (Exception e) {
LOG.warnv("EKS cluster {0} gets no ECR registry mirror: could not copy registries.yaml "
@@ -2001,11 +2018,10 @@ public class EksClusterManager
*/
void linkContainerdCertsDir(String containerId, String clusterName) {
try {
- lifecycleManager.getDockerClient()
- .copyArchiveToContainerCmd(containerId)
- .withTarInputStream(new ByteArrayInputStream(buildContainerdCertsLinkTar()))
- .withRemotePath("/")
- .exec();
+ byte[] tar = buildContainerdCertsLinkTar();
+ RetryingTarCopier.copyStreamed(lifecycleManager.getDockerClient(), containerId, "/",
+ "containerd-certs-link", out -> out.write(tar),
+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);
} catch (Exception e) {
LOG.warnv("EKS cluster {0}: could not link containerd certs.d directory: {1}",
clusterName, e.getMessage());
@@ -2071,12 +2087,10 @@ public class EksClusterManager
writeLocalCopy(Paths.get(config.services().eks().dataPath(), "webhook", clusterName,
POD_IDENTITY_MANIFEST_FILE), manifest, clusterName);
try {
- lifecycleManager.getDockerClient()
- .copyArchiveToContainerCmd(containerId)
- .withTarInputStream(new ByteArrayInputStream(
- tarSingleFile(POD_IDENTITY_MANIFEST_TAR_ENTRY, manifest)))
- .withRemotePath(K3S_DATA_DIR)
- .exec();
+ byte[] tar = tarSingleFile(POD_IDENTITY_MANIFEST_TAR_ENTRY, manifest);
+ RetryingTarCopier.copyStreamed(lifecycleManager.getDockerClient(), containerId, K3S_DATA_DIR,
+ POD_IDENTITY_MANIFEST_FILE, out -> out.write(tar),
+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, archiveRetryBackoffMillis);
LOG.infov("Registered the EKS Pod Identity mutating webhook ({0}) for cluster {1}", url, clusterName);
} catch (Exception e) {
LOG.warnv("EKS cluster {0} gets no pod identity injection: could not copy {1} into the k3s "
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCaBundleTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCaBundleTest.java
index 2f06ea318d..dce357a14c 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCaBundleTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCaBundleTest.java
@@ -16,19 +16,28 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@@ -83,10 +92,19 @@ class ContainerLifecycleManagerCaBundleTest {
@Test
void tlsOnCopiesTheBundleAndAppendsTheTrustEnvAfterTheSpecEnv() throws Exception {
when(tlsConfig.enabled()).thenReturn(true);
- Path bundle = writeBundle();
+ writeBundle();
CreateContainerCmd createCmd = stubCreateContainer();
CopyArchiveToContainerCmd copyCmd = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);
when(dockerClient.copyArchiveToContainerCmd("container-id")).thenReturn(copyCmd);
+ List<InputStream> streams = new ArrayList<>();
+ when(copyCmd.withTarInputStream(any(InputStream.class))).thenAnswer(invocation -> {
+ streams.add(invocation.getArgument(0));
+ return copyCmd;
+ });
+ when(copyCmd.exec()).thenAnswer(invocation -> {
+ assertBundleArchive(streams.getLast());
+ return null;
+ });
assertEquals("container-id", manager().create(specWithEnv(List.of("FOO=bar"))));
@@ -99,16 +117,49 @@ class ContainerLifecycleManagerCaBundleTest {
"AWS_CA_BUNDLE=/etc/floci-ca-bundle.pem"), env);
verify(copyCmd).withRemotePath("/etc");
- ArgumentCaptor<InputStream> tar = ArgumentCaptor.forClass(InputStream.class);
- verify(copyCmd).withTarInputStream(tar.capture());
+ assertEquals(1, streams.size());
verify(copyCmd).exec();
- try (TarArchiveInputStream in = new TarArchiveInputStream(tar.getValue())) {
- TarArchiveEntry entry = in.getNextEntry();
- assertEquals("floci-ca-bundle.pem", entry.getName());
- assertEquals(0644, entry.getMode() & 0777, "world-readable, so a non-root USER can read it");
- assertEquals(Files.readString(bundle), new String(in.readAllBytes(), StandardCharsets.UTF_8));
- assertNull(in.getNextEntry(), "the archive holds only the bundle");
- }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void transientCopyFailurePreservesTheContainerAndTrustWithAFreshTar(boolean consumeCompletely) throws Exception {
+ when(tlsConfig.enabled()).thenReturn(true);
+ writeBundle();
+ CreateContainerCmd createCmd = stubCreateContainer();
+ CopyArchiveToContainerCmd copyCmd = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);
+ when(dockerClient.copyArchiveToContainerCmd("container-id")).thenReturn(copyCmd);
+ List<InputStream> streams = new ArrayList<>();
+ when(copyCmd.withTarInputStream(any(InputStream.class))).thenAnswer(invocation -> {
+ streams.add(invocation.getArgument(0));
+ return copyCmd;
+ });
+ AtomicInteger attempts = new AtomicInteger();
+ when(copyCmd.exec()).thenAnswer(invocation -> {
+ int attempt = attempts.incrementAndGet();
+ InputStream stream = streams.getLast();
+ if (attempt == 1 && !consumeCompletely) {
+ assertEquals(513, stream.readNBytes(513).length, "consume the tar header and part of the CA bundle");
+ } else {
+ assertBundleArchive(stream);
+ }
+ if (attempt == 1) {
+ throw new RuntimeException(new IOException("Connection reset"));
+ }
+ return null;
+ });
+ lenient().when(dockerClient.removeContainerCmd("container-id"))
+ .thenReturn(mock(RemoveContainerCmd.class, RETURNS_SELF));
+
+ assertEquals("container-id", manager().create(specWithEnv(List.of("FOO=bar"))));
+
+ assertEquals(2, attempts.get());
+ assertEquals(2, streams.size());
+ assertNotSame(streams.getFirst(), streams.get(1), "a consumed tar stream cannot be replayed");
+ assertTrue(capturedEnv(createCmd).contains("AWS_CA_BUNDLE=/etc/floci-ca-bundle.pem"));
+ verify(copyCmd, times(2)).withRemotePath("/etc");
+ verify(createCmd).exec();
+ verify(dockerClient, never()).removeContainerCmd(anyString());
}
@Test
@@ -211,8 +262,20 @@ class ContainerLifecycleManagerCaBundleTest {
return Files.writeString(tlsDir.resolve("floci-ca-bundle.pem"), BUNDLE_PEM);
}
+ private static void assertBundleArchive(InputStream stream) throws IOException {
+ byte[] archive = stream.readAllBytes();
+ try (TarArchiveInputStream in = new TarArchiveInputStream(new ByteArrayInputStream(archive))) {
+ TarArchiveEntry entry = in.getNextEntry();
+ assertNotNull(entry, "every attempt must begin with a complete tar header");
+ assertEquals("floci-ca-bundle.pem", entry.getName());
+ assertEquals(0644, entry.getMode() & 0777, "world-readable, so a non-root USER can read it");
+ assertArrayEquals(BUNDLE_PEM.getBytes(StandardCharsets.UTF_8), in.readAllBytes());
+ assertNull(in.getNextEntry(), "the archive holds only the bundle");
+ }
+ }
+
private ContainerLifecycleManager manager() {
- return new ContainerLifecycleManager(dockerClient, imageCacheService, containerDetector, config);
+ return new ContainerLifecycleManager(dockerClient, imageCacheService, containerDetector, config, 0L);
}
private static ContainerSpec specWithEnv(List<String> env) {
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.java
index a5eaac12af..272a45139c 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.java
@@ -2,34 +2,55 @@ package io.github.hectorvent.floci.core.common.docker;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;
+import com.github.dockerjava.api.exception.DockerClientException;
+import com.github.dockerjava.api.exception.NotFoundException;
+import com.github.dockerjava.core.DefaultDockerClientConfig;
+import com.github.dockerjava.core.DockerClientImpl;
+import com.github.dockerjava.core.util.CompressArchiveUtil;
+import com.github.dockerjava.transport.DockerHttpClient;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.NullSource;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor;
+import org.mockito.MockMakers;
+import org.mockito.MockedStatic;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystemException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
+import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
import static org.mockito.Mockito.RETURNS_SELF;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
/**
* The transport-level retry ({@code RetryingDockerHttpClient}) must refuse to replay a one-shot
@@ -47,6 +68,13 @@ class RetryingTarCopierTest {
return new RuntimeException(new IOException("Broken pipe"));
}
+ private static DockerClient dockerClient(DockerHttpClient transport) {
+ return DockerClientImpl.getInstance(DefaultDockerClientConfig.createDefaultConfigBuilder()
+ .withDockerHost("tcp://localhost:2375")
+ .withApiVersion("1.44")
+ .build(), transport);
+ }
+
private static CopyArchiveToContainerCmd failingOnce(DockerClient docker, AtomicInteger calls) {
CopyArchiveToContainerCmd cmd = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);
when(cmd.exec()).thenAnswer(inv -> {
@@ -180,14 +208,128 @@ class RetryingTarCopierTest {
void copyHostResourceRetriesTransientBrokenPipe(@TempDir Path dir) throws IOException {
Path file = dir.resolve("kubeconfig.yaml");
Files.writeString(file, "apiVersion: v1");
- DockerClient docker = mock(DockerClient.class);
- AtomicInteger calls = new AtomicInteger();
- failingOnce(docker, calls);
+ DockerHttpClient transport = mock(DockerHttpClient.class);
+ DockerHttpClient.Response ok = mock(DockerHttpClient.Response.class);
+ when(ok.getStatusCode()).thenReturn(200);
+ when(transport.execute(any())).thenThrow(brokenPipe()).thenReturn(ok);
- RetryingTarCopier.copyHostResource(docker, "c-1", "/var/lib/rancher", file.toString(),
- MAX_ATTEMPTS, 0L);
+ try (DockerClient docker = dockerClient(transport)) {
+ RetryingTarCopier.copyHostResource(docker, "c-1", "/var/lib/rancher", file.toString(),
+ MAX_ATTEMPTS, 0L);
+ }
- assertEquals(2, calls.get());
+ verify(transport, times(2)).execute(any());
+ }
+
+ @ParameterizedTest
+ @NullSource
+ @ValueSource(strings = {"Docker transport failure"})
+ void copyHostResourceStillRetriesTransportClientExceptions(String message, @TempDir Path dir) throws IOException {
+ Path file = dir.resolve("kubeconfig.yaml");
+ Files.writeString(file, "apiVersion: v1");
+ DockerHttpClient transport = mock(DockerHttpClient.class);
+ DockerHttpClient.Response ok = mock(DockerHttpClient.Response.class);
+ when(ok.getStatusCode()).thenReturn(200);
+ when(transport.execute(any()))
+ .thenThrow(new DockerClientException(message, new IOException("Broken pipe"))).thenReturn(ok);
+
+ try (DockerClient docker = dockerClient(transport)) {
+ RetryingTarCopier.copyHostResource(docker, "c-1", "/var/lib/rancher", file.toString(),
+ MAX_ATTEMPTS, 0L);
+ }
+
+ verify(transport, times(2)).execute(any());
+ }
+
+ @Test
+ void copyHostResourceDoesNotRetryLocalArchiveFailure(@TempDir Path dir) throws IOException {
+ Path file = dir.resolve("kubeconfig.yaml");
+ Files.writeString(file, "apiVersion: v1");
+ DockerHttpClient transport = mock(DockerHttpClient.class);
+ IOException archiveFailure = new IOException("No space left on device");
+ AtomicInteger archiveAttempts = new AtomicInteger();
+
+ try (DockerClient docker = dockerClient(transport);
+ MockedStatic<CompressArchiveUtil> archives = mockStatic(CompressArchiveUtil.class,
+ withSettings().mockMaker(MockMakers.INLINE).defaultAnswer(CALLS_REAL_METHODS))) {
+ archives.when(() -> CompressArchiveUtil.tar(any(Path.class), any(Path.class),
+ anyBoolean(), anyBoolean())).thenAnswer(invocation -> {
+ archiveAttempts.incrementAndGet();
+ throw archiveFailure;
+ });
+
+ RuntimeException thrown = assertThrows(RuntimeException.class,
+ () -> RetryingTarCopier.copyHostResource(docker, "c-1", "/var/lib/rancher",
+ file.toString(), MAX_ATTEMPTS, 0L));
+
+ Throwable cause = thrown;
+ while (cause.getCause() != null) {
+ cause = cause.getCause();
+ }
+ assertSame(archiveFailure, cause, "the original local archive failure must survive");
+ verify(transport, never()).execute(any());
+ assertEquals(1, archiveAttempts.get(), "local archive failures cannot recover by retrying Docker");
+ }
+ }
+
+ @Test
+ void copyHostResourceDoesNotRetryLocalArchiveReadFailure(@TempDir Path dir) throws IOException {
+ Path file = dir.resolve("kubeconfig.yaml");
+ Files.writeString(file, "apiVersion: v1");
+ DockerHttpClient transport = mock(DockerHttpClient.class);
+ AtomicInteger archiveAttempts = new AtomicInteger();
+ AtomicReference<Path> archivePath = new AtomicReference<>();
+
+ try (DockerClient docker = dockerClient(transport);
+ MockedStatic<CompressArchiveUtil> archives = mockStatic(CompressArchiveUtil.class,
+ withSettings().mockMaker(MockMakers.INLINE).defaultAnswer(CALLS_REAL_METHODS))) {
+ archives.when(() -> CompressArchiveUtil.tar(any(Path.class), any(Path.class),
+ anyBoolean(), anyBoolean())).thenAnswer(invocation -> {
+ archiveAttempts.incrementAndGet();
+ Path archive = invocation.getArgument(1);
+ archivePath.set(archive);
+ Files.delete(archive);
+ // A symlink loop fails when docker-java opens the archive, before HTTP.
+ Files.createSymbolicLink(archive, archive.getFileName());
+ return null;
+ });
+
+ RuntimeException thrown = assertThrows(RuntimeException.class,
+ () -> RetryingTarCopier.copyHostResource(docker, "c-1", "/var/lib/rancher",
+ file.toString(), MAX_ATTEMPTS, 0L));
+
+ Throwable cause = thrown;
+ while (cause.getCause() != null) {
+ cause = cause.getCause();
+ }
+ FileSystemException readFailure = assertInstanceOf(FileSystemException.class, cause);
+ assertEquals(archivePath.get().toString(), readFailure.getFile());
+ verify(transport, never()).execute(any());
+ assertEquals(1, archiveAttempts.get(), "opening a local archive must not spend the Docker retry budget");
+ }
+ }
+
+ @Test
+ void copyHostResourceDoesNotRetryDaemonRejection(@TempDir Path dir) throws IOException {
+ Path file = dir.resolve("kubeconfig.yaml");
+ Files.writeString(file, "apiVersion: v1");
+ DockerHttpClient transport = mock(DockerHttpClient.class);
+ DockerHttpClient.Response missingContainer = mock(DockerHttpClient.Response.class);
+ when(missingContainer.getStatusCode()).thenReturn(404);
+ when(missingContainer.getHeaders()).thenReturn(Map.of());
+ when(missingContainer.getBody()).thenAnswer(invocation -> new ByteArrayInputStream(
+ "{\"message\":\"No such container: c-1\"}".getBytes(StandardCharsets.UTF_8)));
+ when(transport.execute(any())).thenReturn(missingContainer);
+
+ try (DockerClient docker = dockerClient(transport)) {
+ RuntimeException thrown = assertThrows(RuntimeException.class,
+ () -> RetryingTarCopier.copyHostResource(docker, "c-1", "/var/lib/rancher",
+ file.toString(), MAX_ATTEMPTS, 0L));
+
+ assertInstanceOf(NotFoundException.class, thrown.getCause());
+ }
+
+ verify(transport, times(1)).execute(any());
}
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/UserDataPipelineTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/UserDataPipelineTest.java
index f64b45b31b..f1c7e98e78 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/UserDataPipelineTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/UserDataPipelineTest.java
@@ -1,21 +1,111 @@
package io.github.hectorvent.floci.core.common.docker;
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;
+import com.github.dockerjava.api.exception.NotFoundException;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.GZIPOutputStream;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.RETURNS_SELF;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
class UserDataPipelineTest {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void retriesConsumedScriptUploadBeforeExecutingExactlyOnce(boolean consumeCompletely) {
+ String script = "#!/bin/sh\nprintf 'bootstrap complete\\n'\n";
+ DockerClient docker = mock(DockerClient.class);
+ CopyArchiveToContainerCmd copyCmd = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);
+ when(docker.copyArchiveToContainerCmd("container-id")).thenReturn(copyCmd);
+ List<InputStream> streams = new ArrayList<>();
+ when(copyCmd.withTarInputStream(any(InputStream.class))).thenAnswer(invocation -> {
+ streams.add(invocation.getArgument(0));
+ return copyCmd;
+ });
+ AtomicInteger attempts = new AtomicInteger();
+ when(copyCmd.exec()).thenAnswer(invocation -> {
+ int attempt = attempts.incrementAndGet();
+ InputStream stream = streams.getLast();
+ if (attempt == 1 && !consumeCompletely) {
+ assertEquals(513, stream.readNBytes(513).length, "consume the tar header and part of the script");
+ } else {
+ byte[] archive = stream.readAllBytes();
+ try (TarArchiveInputStream tar = new TarArchiveInputStream(new ByteArrayInputStream(archive))) {
+ TarArchiveEntry entry = tar.getNextEntry();
+ assertNotNull(entry, "every attempt must begin with a complete tar header");
+ assertEquals("user-data.sh", entry.getName());
+ assertEquals(0755, entry.getMode() & 0777);
+ assertArrayEquals(script.getBytes(StandardCharsets.UTF_8), tar.readAllBytes());
+ assertNull(tar.getNextEntry(), "the archive holds only the user-data script");
+ }
+ }
+ if (attempt == 1) {
+ throw new RuntimeException(new IOException("Connection reset"));
+ }
+ return null;
+ });
+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(docker, "container-id", 0L,
+ "bootstrap complete\n", "");
+ List<String> output = new ArrayList<>();
+
+ UserDataPipeline.ExecutionResult result = UserDataPipeline.executeUserData(docker, "container-id",
+ "test instance", script, Duration.ofSeconds(1), output::add, null, 0L);
+
+ assertTrue(result.isSuccess(), result.getFailureMessage());
+ assertEquals(2, attempts.get());
+ assertEquals(2, streams.size());
+ assertNotSame(streams.getFirst(), streams.get(1), "a consumed tar stream cannot be replayed");
+ assertEquals(List.of(List.of(UserDataPipeline.USER_DATA_SCRIPT_PATH)), commands);
+ assertEquals(List.of("bootstrap complete"), output);
+ verify(copyCmd, times(2)).withRemotePath("/var/lib");
+ verify(docker).execStartCmd("exec-container-id");
+ }
+
+ @Test
+ void daemonRejectionDoesNotRetryScriptUploadOrExecuteTheScript() {
+ DockerClient docker = mock(DockerClient.class);
+ CopyArchiveToContainerCmd copyCmd = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);
+ when(docker.copyArchiveToContainerCmd("container-id")).thenReturn(copyCmd);
+ when(copyCmd.exec()).thenThrow(new NotFoundException("Could not find the file /var/lib in container"));
+
+ UserDataPipeline.ExecutionResult result = UserDataPipeline.executeUserData(docker, "container-id",
+ "test instance", "#!/bin/sh\nexit 0\n", Duration.ofSeconds(1), null, null);
+
+ assertFalse(result.isSuccess());
+ assertTrue(result.getFailureMessage().contains("Could not find the file /var/lib in container"));
+ verify(copyCmd).exec();
+ verify(docker, never()).execCreateCmd(anyString());
+ }
+
@Test
void extractsPlainShellScript() {
String script = "#!/bin/bash\necho 'hello world'\n";
diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerArchiveRetryTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerArchiveRetryTest.java
new file mode 100644
index 0000000000..5683a44d2c
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerArchiveRetryTest.java
@@ -0,0 +1,346 @@
+package io.github.hectorvent.floci.services.eks;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;
+import com.github.dockerjava.api.exception.NotFoundException;
+import com.github.dockerjava.core.command.CopyArchiveToContainerCmdImpl;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.config.FlociCertificateAuthority;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;
+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
+import io.github.hectorvent.floci.core.common.docker.PortAllocator;
+import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
+import io.github.hectorvent.floci.services.eks.model.Cluster;
+import io.github.hectorvent.floci.services.eks.model.ClusterIdentity;
+import io.github.hectorvent.floci.services.eks.model.ClusterOidcKey;
+import io.github.hectorvent.floci.services.eks.model.LogSetup;
+import io.github.hectorvent.floci.services.eks.model.Logging;
+import io.github.hectorvent.floci.services.eks.model.OidcIdentity;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.zip.GZIPInputStream;
+
+import static io.github.hectorvent.floci.testing.InlineMocks.inlineMock;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.RETURNS_SELF;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class EksClusterManagerArchiveRetryTest {
+
+ private static final String CONTAINER_ID = "container-1";
+ private static final String CA_PEM = "-----BEGIN CERTIFICATE-----\ntest-only-ca\n-----END CERTIFICATE-----\n";
+ private static final byte[] PRIVATE_KEY = "test-only generated private key\n".getBytes(StandardCharsets.UTF_8);
+ private static final byte[] PUBLIC_KEY = "test-only generated public key\n".getBytes(StandardCharsets.UTF_8);
+
+ @TempDir
+ Path tempDir;
+
+ private final List<ArchiveUpload> uploads = new ArrayList<>();
+ private Function<ArchiveUpload, RuntimeException> failure = upload -> null;
+ private EmulatorConfig.EksServiceConfig eks;
+ private ContainerLifecycleManager lifecycleManager;
+ private EksOidcService oidcService;
+ private EksClusterManager manager;
+
+ @BeforeEach
+ void setUp() {
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);
+ eks = mock(EmulatorConfig.EksServiceConfig.class);
+ EmulatorConfig.TlsConfig tls = mock(EmulatorConfig.TlsConfig.class);
+ when(config.services()).thenReturn(services);
+ when(services.eks()).thenReturn(eks);
+ when(config.tls()).thenReturn(tls);
+ when(config.port()).thenReturn(4566);
+ when(config.defaultAccountId()).thenReturn("000000000000");
+ when(tls.enabled()).thenReturn(true);
+ when(eks.podIdentityWebhook()).thenReturn(true);
+ when(eks.dataPath()).thenReturn(tempDir.toString());
+ when(eks.defaultImage()).thenReturn("rancher/k3s:v1.30.0-k3s1");
+ when(eks.imageTemplate()).thenReturn(Optional.empty());
+ when(eks.dockerNetwork()).thenReturn(Optional.empty());
+ when(eks.endpointMode()).thenReturn("host");
+
+ DockerClient docker = mock(DockerClient.class);
+ CopyArchiveToContainerCmd.Exec copyExec = mock(CopyArchiveToContainerCmd.Exec.class);
+ when(copyExec.exec(any())).thenAnswer(invocation -> {
+ CopyArchiveToContainerCmd command = invocation.getArgument(0);
+ ArchiveUpload upload = consumeArchive(command);
+ uploads.add(upload);
+ RuntimeException copyFailure = failure.apply(upload);
+ if (copyFailure != null) {
+ throw copyFailure;
+ }
+ return null;
+ });
+ when(docker.copyArchiveToContainerCmd(anyString())).thenAnswer(invocation ->
+ new CopyArchiveToContainerCmdImpl(copyExec, invocation.getArgument(0)));
+ lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.getDockerClient()).thenReturn(docker);
+ when(lifecycleManager.create(any())).thenReturn(CONTAINER_ID);
+ when(lifecycleManager.startCreated(any(), any())).thenReturn(new ContainerInfo(CONTAINER_ID, Map.of()));
+
+ ContainerBuilder containerBuilder = mock(ContainerBuilder.class);
+ ContainerBuilder.Builder builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
+ when(containerBuilder.newContainer(anyString())).thenReturn(builder);
+ when(builder.build()).thenReturn(new ContainerSpec("test-image"));
+ DockerHostResolver dockerHostResolver = mock(DockerHostResolver.class);
+ when(dockerHostResolver.resolve()).thenReturn("host.docker.internal");
+ RegionResolver regionResolver = mock(RegionResolver.class);
+ when(regionResolver.getAccountId()).thenReturn("000000000000");
+ when(regionResolver.getDefaultRegion()).thenReturn("us-east-1");
+ FlociCertificateAuthority certificateAuthority = inlineMock(FlociCertificateAuthority.class);
+ when(certificateAuthority.caPem()).thenReturn(CA_PEM);
+ oidcService = mock(EksOidcService.class);
+
+ manager = new EksClusterManager(containerBuilder, lifecycleManager,
+ mock(ContainerDetector.class), mock(PortAllocator.class), dockerHostResolver,
+ mock(EcrRegistryManager.class), config, regionResolver, null, oidcService, certificateAuthority,
+ null, null, 0L);
+ }
+
+ @Test
+ void auditPolicyRetriesAfterTheFirstArchiveIsConsumed() throws IOException {
+ Path policy = tempDir.resolve("audit-policy.yaml");
+ byte[] content = EksClusterManager.buildAuditPolicy().getBytes(StandardCharsets.UTF_8);
+ Files.write(policy, content);
+ failFirstUploadOf("audit-policy.yaml");
+
+ manager.copyAuditPolicyIntoContainer(CONTAINER_ID, policy.toString(), "demo");
+
+ assertEquals(2, uploads.size(), "a consumed audit policy archive must be rebuilt after a broken pipe");
+ for (ArchiveUpload upload : uploads) {
+ assertHostFileUpload(upload, policy, content);
+ }
+ }
+
+ @Test
+ void signingPrivateKeyRetriesBeforeUploadingThePublicKey() throws IOException {
+ EksClusterManager.SigningKeyFiles keys = signingKeyFiles();
+ failFirstUploadOf("sa-signing-key.pem");
+
+ manager.copySigningKeysIntoContainer(CONTAINER_ID, keys, "demo");
+
+ assertEquals(3, uploads.size(), "the private key must recover before the public key is uploaded");
+ assertHostFileUpload(uploads.get(0), keys.signingKeyPath(), PRIVATE_KEY);
+ assertHostFileUpload(uploads.get(1), keys.signingKeyPath(), PRIVATE_KEY);
+ assertHostFileUpload(uploads.get(2), keys.publicKeyPath(), PUBLIC_KEY);
+ assertSigningKeySourcesRemainPrivate(keys);
+ }
+
+ @Test
+ void signingPublicKeyRetriesWithoutUploadingTheSuccessfulPrivateKeyAgain() throws IOException {
+ EksClusterManager.SigningKeyFiles keys = signingKeyFiles();
+ failFirstUploadOf("sa-public-key.pem");
+
+ manager.copySigningKeysIntoContainer(CONTAINER_ID, keys, "demo");
+
+ assertEquals(3, uploads.size(), "only the failed public key upload must be retried");
+ assertHostFileUpload(uploads.get(0), keys.signingKeyPath(), PRIVATE_KEY);
+ assertHostFileUpload(uploads.get(1), keys.publicKeyPath(), PUBLIC_KEY);
+ assertHostFileUpload(uploads.get(2), keys.publicKeyPath(), PUBLIC_KEY);
+ assertSigningKeySourcesRemainPrivate(keys);
+ }
+
+ @Test
+ void dockerJavaDirectHostResourceCopyUses0644ForPrivateSourceFiles() throws IOException {
+ EksClusterManager.SigningKeyFiles keys = signingKeyFiles();
+
+ lifecycleManager.getDockerClient().copyArchiveToContainerCmd(CONTAINER_ID)
+ .withHostResource(keys.signingKeyPath().toString())
+ .withRemotePath("/etc")
+ .exec();
+
+ assertEquals(1, uploads.size());
+ assertHostFileUpload(uploads.getFirst(), keys.signingKeyPath(), PRIVATE_KEY);
+ assertSigningKeySourcesRemainPrivate(keys);
+ }
+
+ @Test
+ void containerdLinkRetriesWithEveryDirectoryAndTheSymlinkIntact() {
+ failFirstUploadOf("etc/containerd/certs.d");
+
+ manager.linkContainerdCertsDir(CONTAINER_ID, "demo");
+
+ assertEquals(2, uploads.size(), "the entire multi-entry archive must survive the retry");
+ for (ArchiveUpload upload : uploads) {
+ assertEquals(CONTAINER_ID, upload.containerId());
+ assertEquals("/", upload.remotePath());
+ assertEquals(3, upload.entries().size());
+ assertDirectory(upload.entries().get(0), "var/lib/rancher/k3s/agent/etc/containerd/certs.d/");
+ assertDirectory(upload.entries().get(1), "etc/containerd/");
+ ArchiveEntry link = upload.entries().get(2);
+ assertEquals("etc/containerd/certs.d", link.name());
+ assertTrue(link.symbolicLink());
+ assertFalse(link.directory());
+ assertEquals(0777, link.mode());
+ assertEquals("/var/lib/rancher/k3s/agent/etc/containerd/certs.d", link.linkName());
+ assertArrayEquals(new byte[0], link.content());
+ }
+ }
+
+ @Test
+ void podIdentityManifestRetriesWithItsFullContentAndMode() {
+ failFirstUploadOf("server/manifests/floci-eks-pod-identity.yaml");
+ byte[] expected = EksClusterManager.buildPodIdentityWebhookConfiguration(
+ "https://host.docker.internal:4566/_floci/eks/clusters/demo/pod-identity-webhook/scope/000000000000",
+ CA_PEM).getBytes(StandardCharsets.UTF_8);
+
+ manager.registerPodIdentityWebhook(CONTAINER_ID, cluster());
+
+ assertEquals(2, uploads.size(), "the pod identity manifest must be rebuilt after the consumed upload fails");
+ for (ArchiveUpload upload : uploads) {
+ assertFileUpload(upload, "/var/lib/rancher/k3s", "server/manifests/floci-eks-pod-identity.yaml", expected);
+ }
+ }
+
+ @Test
+ void publicKeyDaemonRejectionDoesNotRetryEitherKeyOrPropagate() throws IOException {
+ EksClusterManager.SigningKeyFiles keys = signingKeyFiles();
+ failure = upload -> containsEntry(upload, "sa-public-key.pem")
+ ? new NotFoundException("No such container: " + CONTAINER_ID) : null;
+
+ assertDoesNotThrow(() -> manager.copySigningKeysIntoContainer(CONTAINER_ID, keys, "demo"));
+
+ assertEquals(2, uploads.size());
+ assertHostFileUpload(uploads.get(0), keys.signingKeyPath(), PRIVATE_KEY);
+ assertHostFileUpload(uploads.get(1), keys.publicKeyPath(), PUBLIC_KEY);
+ assertSigningKeySourcesRemainPrivate(keys);
+ }
+
+ @Test
+ void archiveDaemonRejectionsAreNotRetriedAndLeaveTheClusterRunning() {
+ failure = upload -> new NotFoundException("No such container: " + CONTAINER_ID);
+ when(eks.irsaSigningKey()).thenReturn(true);
+ Cluster cluster = cluster();
+ cluster.setLogging(new Logging(List.of(new LogSetup(List.of("audit"), true))));
+ ClusterOidcKey key = new ClusterOidcKey("https://test.invalid/issuer", "fixture", "public", "private");
+ when(oidcService.ensureKeyForAccount("000000000000", "demo", key.getIssuer())).thenReturn(key);
+ when(oidcService.exportSigningKeyPem(key)).thenReturn(new String(PRIVATE_KEY, StandardCharsets.UTF_8));
+ when(oidcService.exportPublicKeyPem(key)).thenReturn(new String(PUBLIC_KEY, StandardCharsets.UTF_8));
+
+ assertDoesNotThrow(() -> manager.startCluster(cluster));
+
+ assertEquals(List.of("audit-policy.yaml", "etc/containerd/certs.d",
+ "server/manifests/floci-eks-pod-identity.yaml", "sa-signing-key.pem"),
+ uploads.stream().map(upload -> upload.entries().getLast().name()).toList(),
+ "each rejected feature is attempted once and startup continues with reduced functionality");
+ verify(lifecycleManager).startCreated(any(), any());
+ assertEquals(CONTAINER_ID, cluster.getContainerId());
+ }
+
+ private EksClusterManager.SigningKeyFiles signingKeyFiles() throws IOException {
+ Path signingKey = tempDir.resolve("sa-signing-key.pem");
+ Path publicKey = tempDir.resolve("sa-public-key.pem");
+ Files.write(signingKey, PRIVATE_KEY);
+ Files.write(publicKey, PUBLIC_KEY);
+ if (Files.getFileStore(tempDir).supportsFileAttributeView("posix")) {
+ Files.setPosixFilePermissions(signingKey, PosixFilePermissions.fromString("rw-------"));
+ Files.setPosixFilePermissions(publicKey, PosixFilePermissions.fromString("rw-------"));
+ }
+ return new EksClusterManager.SigningKeyFiles(signingKey, publicKey);
+ }
+
+ private static void assertSigningKeySourcesRemainPrivate(EksClusterManager.SigningKeyFiles keys) throws IOException {
+ if (Files.getFileStore(keys.signingKeyPath()).supportsFileAttributeView("posix")) {
+ assertEquals(PosixFilePermissions.fromString("rw-------"), Files.getPosixFilePermissions(keys.signingKeyPath()));
+ assertEquals(PosixFilePermissions.fromString("rw-------"), Files.getPosixFilePermissions(keys.publicKeyPath()));
+ }
+ }
+
+ private static Cluster cluster() {
+ Cluster cluster = new Cluster();
+ cluster.setName("demo");
+ cluster.setAccountId("000000000000");
+ cluster.setIdentity(new ClusterIdentity(new OidcIdentity("https://test.invalid/issuer")));
+ return cluster;
+ }
+
+ private void failFirstUploadOf(String name) {
+ AtomicInteger attempts = new AtomicInteger();
+ failure = upload -> containsEntry(upload, name) && attempts.incrementAndGet() == 1
+ ? new RuntimeException(new IOException("Broken pipe")) : null;
+ }
+
+ private static boolean containsEntry(ArchiveUpload upload, String name) {
+ return upload.entries().stream().anyMatch(entry -> entry.name().equals(name));
+ }
+
+ private static ArchiveUpload consumeArchive(CopyArchiveToContainerCmd command) throws IOException {
+ InputStream body = command.getTarInputStream();
+ if (command.getHostResource() != null) {
+ body = new GZIPInputStream(body);
+ }
+ List<ArchiveEntry> entries = new ArrayList<>();
+ try (TarArchiveInputStream tar = new TarArchiveInputStream(body)) {
+ TarArchiveEntry entry;
+ while ((entry = tar.getNextEntry()) != null) {
+ entries.add(new ArchiveEntry(entry.getName(), entry.getMode() & 07777,
+ entry.isDirectory(), entry.isSymbolicLink(), entry.getLinkName(), tar.readAllBytes()));
+ }
+ body.transferTo(OutputStream.nullOutputStream());
+ }
+ return new ArchiveUpload(command.getContainerId(), command.getRemotePath(), command.getHostResource(), entries);
+ }
+
+ private static void assertHostFileUpload(ArchiveUpload upload, Path source, byte[] content) {
+ assertEquals(source.toString(), upload.hostResource());
+ // docker-java's File-based tar entries default to 0644 even when the source file is protected with 0600.
+ assertFileUpload(upload, "/etc", source.getFileName().toString(), content);
+ }
+
+ private static void assertFileUpload(ArchiveUpload upload, String remotePath, String name, byte[] content) {
+ assertEquals(CONTAINER_ID, upload.containerId());
+ assertEquals(remotePath, upload.remotePath());
+ assertEquals(1, upload.entries().size());
+ ArchiveEntry entry = upload.entries().getFirst();
+ assertEquals(name, entry.name());
+ assertEquals(0644, entry.mode());
+ assertFalse(entry.directory());
+ assertFalse(entry.symbolicLink());
+ assertArrayEquals(content, entry.content());
+ }
+
+ private static void assertDirectory(ArchiveEntry entry, String name) {
+ assertEquals(name, entry.name());
+ assertTrue(entry.directory());
+ assertFalse(entry.symbolicLink());
+ assertEquals(0755, entry.mode());
+ assertArrayEquals(new byte[0], entry.content());
+ }
+
+ private record ArchiveUpload(String containerId, String remotePath, String hostResource, List<ArchiveEntry> entries) {}
+
+ private record ArchiveEntry(String name, int mode, boolean directory, boolean symbolicLink,
+ String linkName, byte[] content) {}
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
index cb6e3e796f..0590e6dce1 100644
--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
@@ -62,6 +62,8 @@ import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
@@ -1330,15 +1332,23 @@ class EksClusterManagerTest {
@Test
void copiesArchiveWithTargetDirectoryAndSymlinkAtContainerRoot() throws Exception {
+ ArgumentCaptor<InputStream> archive = ArgumentCaptor.forClass(InputStream.class);
+ when(copyCmd.withTarInputStream(archive.capture())).thenReturn(copyCmd);
+ ByteArrayOutputStream archiveBytes = new ByteArrayOutputStream();
+ when(copyCmd.exec()).thenAnswer(invocation -> {
+ archive.getValue().transferTo(archiveBytes);
+ return null;
+ });
+
manager.linkContainerdCertsDir("container-1", "demo");
verify(dockerClient).copyArchiveToContainerCmd("container-1");
verify(copyCmd).withRemotePath("/");
verify(copyCmd).exec();
- ArgumentCaptor<InputStream> archive = ArgumentCaptor.forClass(InputStream.class);
- verify(copyCmd).withTarInputStream(archive.capture());
- try (TarArchiveInputStream tar = new TarArchiveInputStream(archive.getValue())) {
+ verify(copyCmd).withTarInputStream(any());
+ try (TarArchiveInputStream tar = new TarArchiveInputStream(
+ new ByteArrayInputStream(archiveBytes.toByteArray()))) {
TarArchiveEntry targetDir = tar.getNextEntry();
assertNotNull(targetDir);
assertEquals("var/lib/rancher/k3s/agent/etc/containerd/certs.d/", targetDir.getName());
diff --git a/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2IntegrationTest.java
index 3e4570e571..dc2c446d99 100644
--- a/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2IntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2IntegrationTest.java
@@ -481,27 +481,33 @@ class ResourceExplorer2IntegrationTest {
}
@Test
- void searchWithEmptyQueryReturnsAllResources() {
- Response response = given()
+ void searchWithEmptyQueryReturnsEveryResourceInItsView() {
+ String bucketArn = Arn.global(AwsRegions.partitionFor("us-east-1"), "s3", "", "re2-test-bucket")
+ .toString();
+ String viewArn = trackView(given()
.header("Authorization", AUTH)
.contentType("application/json")
- .body("""
- {"QueryString": ""}
- """)
+ .body(Map.of("ViewName", "empty-query-fixture-view",
+ "Filters", Map.of("FilterString", "id:" + bucketArn)))
+ .when()
+ .post("/CreateView")
+ .then()
+ .statusCode(200)
+ .extract().path("View.ViewArn"));
+
+ given()
+ .header("Authorization", AUTH)
+ .contentType("application/json")
+ .body(Map.of("QueryString", "", "ViewArn", viewArn))
.when()
.post("/Search")
.then()
.statusCode(200)
- .body("Count.TotalResources", greaterThanOrEqualTo(0))
- .body("Resources", notNullValue())
- .extract().response();
-
- // Unscoped, so how many resources the shared emulator holds is not knowable here.
- // Complete says whether the 1000-result cap bit, which is exactly what TotalResources
- // reaching the cap means — assert that relationship rather than a fixed value.
- int total = response.path("Count.TotalResources");
- boolean complete = response.path("Count.Complete");
- assertEquals(total < 1000, complete);
+ .body("Count.TotalResources", equalTo(1))
+ .body("Count.Complete", equalTo(true))
+ .body("Resources.size()", equalTo(1))
+ .body("Resources[0].Arn", equalTo(bucketArn))
+ .body("ViewArn", equalTo(viewArn));
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2ListVsSearchCapTest.java b/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2ListVsSearchCapTest.java
index fb17f2b89e..8c40fd12ac 100644
--- a/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2ListVsSearchCapTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/resourceexplorer2/ResourceExplorer2ListVsSearchCapTest.java
@@ -14,6 +14,8 @@ import io.github.hectorvent.floci.services.resourceexplorer2.model.View;
import io.quarkus.runtime.StartupEvent;
import jakarta.enterprise.inject.Instance;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
import java.time.Instant;
import java.util.ArrayList;
@@ -134,6 +136,17 @@ class ResourceExplorer2ListVsSearchCapTest {
assertEquals(1000, lastTotal);
}
+ @ParameterizedTest
+ @CsvSource({"999,999,true", "1000,1000,true", "1001,1000,false"})
+ void searchCountDistinguishesExactCapFromTruncation(int resourceCount, int expectedTotal, boolean complete) {
+ ObjectNode result = newServiceWith(resourceCount).search("", 1000, null, null, REGION);
+
+ assertEquals(expectedTotal, result.path("Resources").size());
+ assertEquals(expectedTotal, result.path("Count").path("TotalResources").asInt());
+ assertEquals(complete, result.path("Count").path("Complete").asBoolean());
+ assertFalse(result.has("NextToken"));
+ }
+
@Test
void oneThrowingProviderIsSkippedAndHealthyProvidersStillSurface() {
// Discovery aggregates every provider. A single provider blowing up (bad stored data, a bug)