Light, fluffy, and always free - The AWS Local Emulator alternative
feat(eks): add option to disable k3s's bundled CNI (#1793)
Mike Attara444e4a6e89a1
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex a3913efa89..06b1240505 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -1514,6 +1514,18 @@ public interface EmulatorConfig {*/@WithDefault("true")boolean ecrRegistryMirror();++ /**+ * When true, starts k3s with {@code --flannel-backend=none --disable-network-policy+ * --disable-kube-proxy} instead of its bundled networking stack. k3s's default flannel CNI+ * and kube-proxy run embedded in the k3s server process itself (not separate, killable+ * DaemonSets), so a real CNI (e.g. Cilium) can only cleanly take over if k3s never starts+ * its own in the first place — there is no way to evict them after the fact. CoreDNS,+ * local-path-provisioner, and metrics-server are unaffected; they don't depend on which CNI+ * is in place.+ */+ @WithDefault("false")+ boolean disableCni();}interface InitHooksConfig {diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.javaindex e77c596369..205ee525aa 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java@@ -111,9 +111,7 @@ public class EksClusterManager {// filesystem, so chmod works correctly and data persists across container restarts.String volumeName = ContainerStorageHelper.resourceName(config, "eks", null, cluster.getName());- List<String> serverArgs = new ArrayList<>(List.of("server",- "--disable=traefik",- "--tls-san=localhost"));+ List<String> serverArgs = buildServerArgs(config.services().eks().disableCni());ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image).withName(containerName)@@ -141,7 +139,20 @@ public class EksClusterManager {}}- ContainerSpec spec = specBuilder.withCmd(serverArgs).build();+ if (config.services().eks().disableCni()) {+ // A container's /sys mount defaults to private propagation, which breaks+ // Cilium's BPF filesystem mount ("mounted on /sys but it is not a shared or+ // slave mount") — real EKS/kubeadm nodes don't hit this since they're VMs,+ // not nested containers. `mount --make-rshared /` before k3s starts fixes+ // it; kind's own node image runs the same fix in its entrypoint for the+ // same reason. RSHARE_ENTRYPOINT is POSIX sh-compatible (no bashisms) — the+ // k3s image has no bash, only busybox sh.+ specBuilder.withEntrypoint(RSHARE_ENTRYPOINT);+ specBuilder.withCmd(buildRshareWrappedCmd(serverArgs));+ } else {+ specBuilder.withCmd(serverArgs);+ }+ ContainerSpec spec = specBuilder.build();// create -> inject webhook kubeconfig -> start, so the file exists before the API server boots.String containerId = lifecycleManager.create(spec);@@ -248,6 +259,47 @@ public class EksClusterManager {LOG.infov("Stopped k3s container for cluster {0}", cluster.getName());}+ /**+ * Builds the k3s {@code server} command-line args. When {@code disableCni} is true, flannel,+ * k3s's default network policy controller, and kube-proxy are all disabled up front — see the+ * {@code disableCni} config javadoc for why this must happen at startup, not after the fact.+ */+ static List<String> buildServerArgs(boolean disableCni) {+ List<String> serverArgs = new ArrayList<>(List.of("server",+ "--disable=traefik",+ "--tls-san=localhost"));+ if (disableCni) {+ serverArgs.add("--flannel-backend=none");+ serverArgs.add("--disable-network-policy");+ serverArgs.add("--disable-kube-proxy");+ }+ return serverArgs;+ }++ /**+ * Overrides the image's default {@code ["/bin/k3s"]} entrypoint so a {@code mount+ * --make-rshared /} can run immediately before k3s starts (see the disableCni branch+ * in {@link #startCluster} for why). POSIX sh-compatible — the k3s image has no bash.+ * A failed mount is logged to stderr rather than silently ignored, since it means the+ * external CNI's BPF filesystem mount will fail later in a much more confusing way.+ */+ static final List<String> RSHARE_ENTRYPOINT = List.of("sh", "-c",+ "mount --make-rshared / || echo 'floci: WARN: mount --make-rshared / failed; "+ + "external CNI may not work' >&2; exec /bin/k3s \"$@\"");++ /**+ * Builds the CMD to pair with {@link #RSHARE_ENTRYPOINT}: an unused $0 placeholder+ * followed by the real k3s server args, so the entrypoint's "$@" expands to exactly+ * {@code serverArgs} — the same args {@code withCmd(serverArgs)} would pass directly+ * when the entrypoint isn't overridden.+ */+ static List<String> buildRshareWrappedCmd(List<String> serverArgs) {+ List<String> wrappedCmd = new ArrayList<>();+ wrappedCmd.add("floci-k3s");+ wrappedCmd.addAll(serverArgs);+ return wrappedCmd;+ }+/*** Resolves the public {@code describe-cluster} endpoint. Returns the container DNS name only when* Floci runs in a container and {@code endpoint-mode=network}; otherwise the host-reachablediff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex debf832e56..3c59def2f4 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -375,6 +375,7 @@ floci:endpoint-mode: host # host = https://localhost:<hostPort> (host-reachable); network = container DNS nameiam-auth-webhook: true # wire a token-auth webhook into k3s so `aws eks get-token` worksecr-registry-mirror: true # inject a registries.yaml so pods can pull images pushed to Floci ECR+ disable-cni: false # start k3s without its bundled flannel/kube-proxy so a real CNI (e.g. Cilium) can take overelbv2:enabled: truemock: falsediff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.javaindex 13696dd458..3b6ac87134 100644--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java@@ -102,6 +102,58 @@ class EksClusterManagerTest {assertTrue(yaml.contains("- \"http://my-registry:5000\""));}+ @Test+ void serverArgsOmitCniFlagsByDefault() {+ List<String> args = EksClusterManager.buildServerArgs(false);++ assertTrue(args.contains("--disable=traefik"));+ assertFalse(args.contains("--flannel-backend=none"));+ assertFalse(args.contains("--disable-network-policy"));+ assertFalse(args.contains("--disable-kube-proxy"));+ }++ @Test+ void serverArgsDisableFlannelAndKubeProxyWhenRequested() {+ List<String> args = EksClusterManager.buildServerArgs(true);++ assertTrue(args.contains("--flannel-backend=none"));+ assertTrue(args.contains("--disable-network-policy"));+ assertTrue(args.contains("--disable-kube-proxy"));+ // Base args must still be present — disableCni only adds flags, never replaces them.+ assertTrue(args.contains("--disable=traefik"));+ assertTrue(args.contains("--tls-san=localhost"));+ }++ @Test+ void rshareEntrypointIsPosixShCompatible() {+ String script = EksClusterManager.RSHARE_ENTRYPOINT.get(2);++ assertEquals(List.of("sh", "-c"), EksClusterManager.RSHARE_ENTRYPOINT.subList(0, 2));+ assertTrue(script.contains("mount --make-rshared /"));+ assertTrue(script.contains("exec /bin/k3s"));+ assertFalse(script.contains("bash"), "the k3s image has no bash, only busybox sh");+ }++ @Test+ void rshareEntrypointWarnsOnStderrWhenMountFails() {+ String script = EksClusterManager.RSHARE_ENTRYPOINT.get(2);++ // A failed mount must be surfaced, not silently swallowed, so a later CNI failure+ // is traceable back to this step instead of looking unrelated.+ assertTrue(script.contains("|| echo"));+ assertTrue(script.contains(">&2"));+ assertFalse(script.contains("2>/dev/null"));+ }++ @Test+ void rshareWrappedCmdPreservesServerArgsAfterThePlaceholder() {+ List<String> serverArgs = EksClusterManager.buildServerArgs(true);+ List<String> wrapped = EksClusterManager.buildRshareWrappedCmd(serverArgs);++ // First element is an unused $0 placeholder consumed by sh -c, not part of serverArgs.+ assertEquals(serverArgs, wrapped.subList(1, wrapped.size()));+ }+/** Mirror-injection guard behavior, without a Docker daemon. */@Nestedclass InjectEcrRegistryMirror {