Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(eks): add option to disable k3s's bundled CNI (#1793)

Mike Attara444e4a6e89a1
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index a3913efa89..06b1240505 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -1514,6 +1514,18 @@ public interface EmulatorConfig {
*/
@WithDefault("true")
boolean ecrRegistryMirror();
+
+ /**
+ * When true, starts k3s with {@code --flannel-backend=none --disable-network-policy
+ * --disable-kube-proxy} instead of its bundled networking stack. k3s's default flannel CNI
+ * and kube-proxy run embedded in the k3s server process itself (not separate, killable
+ * DaemonSets), so a real CNI (e.g. Cilium) can only cleanly take over if k3s never starts
+ * its own in the first place — there is no way to evict them after the fact. CoreDNS,
+ * local-path-provisioner, and metrics-server are unaffected; they don't depend on which CNI
+ * is in place.
+ */
+ @WithDefault("false")
+ boolean disableCni();
}
interface InitHooksConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
index e77c596369..205ee525aa 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
@@ -111,9 +111,7 @@ public class EksClusterManager {
// filesystem, so chmod works correctly and data persists across container restarts.
String volumeName = ContainerStorageHelper.resourceName(config, "eks", null, cluster.getName());
- List<String> serverArgs = new ArrayList<>(List.of("server",
- "--disable=traefik",
- "--tls-san=localhost"));
+ List<String> serverArgs = buildServerArgs(config.services().eks().disableCni());
ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image)
.withName(containerName)
@@ -141,7 +139,20 @@ public class EksClusterManager {
}
}
- ContainerSpec spec = specBuilder.withCmd(serverArgs).build();
+ if (config.services().eks().disableCni()) {
+ // A container's /sys mount defaults to private propagation, which breaks
+ // Cilium's BPF filesystem mount ("mounted on /sys but it is not a shared or
+ // slave mount") — real EKS/kubeadm nodes don't hit this since they're VMs,
+ // not nested containers. `mount --make-rshared /` before k3s starts fixes
+ // it; kind's own node image runs the same fix in its entrypoint for the
+ // same reason. RSHARE_ENTRYPOINT is POSIX sh-compatible (no bashisms) — the
+ // k3s image has no bash, only busybox sh.
+ specBuilder.withEntrypoint(RSHARE_ENTRYPOINT);
+ specBuilder.withCmd(buildRshareWrappedCmd(serverArgs));
+ } else {
+ specBuilder.withCmd(serverArgs);
+ }
+ ContainerSpec spec = specBuilder.build();
// create -> inject webhook kubeconfig -> start, so the file exists before the API server boots.
String containerId = lifecycleManager.create(spec);
@@ -248,6 +259,47 @@ public class EksClusterManager {
LOG.infov("Stopped k3s container for cluster {0}", cluster.getName());
}
+ /**
+ * Builds the k3s {@code server} command-line args. When {@code disableCni} is true, flannel,
+ * k3s's default network policy controller, and kube-proxy are all disabled up front — see the
+ * {@code disableCni} config javadoc for why this must happen at startup, not after the fact.
+ */
+ static List<String> buildServerArgs(boolean disableCni) {
+ List<String> serverArgs = new ArrayList<>(List.of("server",
+ "--disable=traefik",
+ "--tls-san=localhost"));
+ if (disableCni) {
+ serverArgs.add("--flannel-backend=none");
+ serverArgs.add("--disable-network-policy");
+ serverArgs.add("--disable-kube-proxy");
+ }
+ return serverArgs;
+ }
+
+ /**
+ * Overrides the image's default {@code ["/bin/k3s"]} entrypoint so a {@code mount
+ * --make-rshared /} can run immediately before k3s starts (see the disableCni branch
+ * in {@link #startCluster} for why). POSIX sh-compatible — the k3s image has no bash.
+ * A failed mount is logged to stderr rather than silently ignored, since it means the
+ * external CNI's BPF filesystem mount will fail later in a much more confusing way.
+ */
+ static final List<String> RSHARE_ENTRYPOINT = List.of("sh", "-c",
+ "mount --make-rshared / || echo 'floci: WARN: mount --make-rshared / failed; "
+ + "external CNI may not work' >&2; exec /bin/k3s \"$@\"");
+
+ /**
+ * Builds the CMD to pair with {@link #RSHARE_ENTRYPOINT}: an unused $0 placeholder
+ * followed by the real k3s server args, so the entrypoint's "$@" expands to exactly
+ * {@code serverArgs} — the same args {@code withCmd(serverArgs)} would pass directly
+ * when the entrypoint isn't overridden.
+ */
+ static List<String> buildRshareWrappedCmd(List<String> serverArgs) {
+ List<String> wrappedCmd = new ArrayList<>();
+ wrappedCmd.add("floci-k3s");
+ wrappedCmd.addAll(serverArgs);
+ return wrappedCmd;
+ }
+
/**
* Resolves the public {@code describe-cluster} endpoint. Returns the container DNS name only when
* Floci runs in a container and {@code endpoint-mode=network}; otherwise the host-reachable
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index debf832e56..3c59def2f4 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -375,6 +375,7 @@ floci:
endpoint-mode: host # host = https://localhost:<hostPort> (host-reachable); network = container DNS name
iam-auth-webhook: true # wire a token-auth webhook into k3s so `aws eks get-token` works
ecr-registry-mirror: true # inject a registries.yaml so pods can pull images pushed to Floci ECR
+ disable-cni: false # start k3s without its bundled flannel/kube-proxy so a real CNI (e.g. Cilium) can take over
elbv2:
enabled: true
mock: false
diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
index 13696dd458..3b6ac87134 100644
--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
@@ -102,6 +102,58 @@ class EksClusterManagerTest {
assertTrue(yaml.contains("- \"http://my-registry:5000\""));
}
+ @Test
+ void serverArgsOmitCniFlagsByDefault() {
+ List<String> args = EksClusterManager.buildServerArgs(false);
+
+ assertTrue(args.contains("--disable=traefik"));
+ assertFalse(args.contains("--flannel-backend=none"));
+ assertFalse(args.contains("--disable-network-policy"));
+ assertFalse(args.contains("--disable-kube-proxy"));
+ }
+
+ @Test
+ void serverArgsDisableFlannelAndKubeProxyWhenRequested() {
+ List<String> args = EksClusterManager.buildServerArgs(true);
+
+ assertTrue(args.contains("--flannel-backend=none"));
+ assertTrue(args.contains("--disable-network-policy"));
+ assertTrue(args.contains("--disable-kube-proxy"));
+ // Base args must still be present — disableCni only adds flags, never replaces them.
+ assertTrue(args.contains("--disable=traefik"));
+ assertTrue(args.contains("--tls-san=localhost"));
+ }
+
+ @Test
+ void rshareEntrypointIsPosixShCompatible() {
+ String script = EksClusterManager.RSHARE_ENTRYPOINT.get(2);
+
+ assertEquals(List.of("sh", "-c"), EksClusterManager.RSHARE_ENTRYPOINT.subList(0, 2));
+ assertTrue(script.contains("mount --make-rshared /"));
+ assertTrue(script.contains("exec /bin/k3s"));
+ assertFalse(script.contains("bash"), "the k3s image has no bash, only busybox sh");
+ }
+
+ @Test
+ void rshareEntrypointWarnsOnStderrWhenMountFails() {
+ String script = EksClusterManager.RSHARE_ENTRYPOINT.get(2);
+
+ // A failed mount must be surfaced, not silently swallowed, so a later CNI failure
+ // is traceable back to this step instead of looking unrelated.
+ assertTrue(script.contains("|| echo"));
+ assertTrue(script.contains(">&2"));
+ assertFalse(script.contains("2>/dev/null"));
+ }
+
+ @Test
+ void rshareWrappedCmdPreservesServerArgsAfterThePlaceholder() {
+ List<String> serverArgs = EksClusterManager.buildServerArgs(true);
+ List<String> wrapped = EksClusterManager.buildRshareWrappedCmd(serverArgs);
+
+ // First element is an unused $0 placeholder consumed by sh -c, not part of serverArgs.
+ assertEquals(serverArgs, wrapped.subList(1, wrapped.size()));
+ }
+
/** Mirror-injection guard behavior, without a Docker daemon. */
@Nested
class InjectEcrRegistryMirror {