Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(eks): deliver audit logs from the cluster API server (#4146)

Simone Primarosaf5c63ff39851
diff --git a/docs/services/eks.md b/docs/services/eks.md
index 2183672dfb..e65ad629d3 100644
--- a/docs/services/eks.md
+++ b/docs/services/eks.md
@@ -227,9 +227,12 @@ EKS clusters support configuring KMS envelope encryption for secrets and control
- **Response normalization**: AWS EKS always returns the status of all five log types. Enabled types are returned first (`enabled: true`), followed by disabled types (`enabled: false`).
- **Default logging**: When omitted or empty at creation time, Floci returns all five log types disabled in a single entry (`enabled: false`).
- **Backfill**: Existing persisted clusters created before this feature was introduced are automatically backfilled on startup with default disabled logging.
-- **CloudWatch Logs delivery**: When the `api` log type is enabled on a cluster, Floci creates the CloudWatch Logs log group `/aws/eks/<cluster-name>/cluster` and streams control plane container output to a log stream named `kube-apiserver-<hash>`, where `<hash>` is the first 32 characters of the container ID. When `api` logging is disabled or omitted, no log group is created and container logs are not streamed.
-- **Single-stream deviation**: AWS EKS provisions separate log streams for each component (`kube-apiserver-*`, `kube-controller-manager-*`, `kube-scheduler-*`, `authenticator-*`). Because Floci runs clusters on k3s, which embeds the Kubernetes API server, controller manager, and scheduler within a single process, control plane container logs are delivered to the single `kube-apiserver-<hash>` stream when the `api` log type is enabled. Other control plane log types (`audit`, `authenticator`, `controllerManager`, `scheduler`) do not provision separate streams.
-- **Audit and authenticator logs**: Audit logging is not configured in k3s. Authenticator webhook authentication events are logged directly by Floci.
+- **CloudWatch Logs delivery**:
+ - When the `api` log type is enabled on a cluster, Floci creates the CloudWatch Logs log group `/aws/eks/<cluster-name>/cluster` and streams control plane container output to a log stream named `kube-apiserver-<hash>`, where `<hash>` is the first 32 characters of the container ID.
+ - When the `audit` log type is enabled on a cluster, Floci configures k3s with the Amazon EKS control plane audit policy, writes audit logs to `/var/log/audit.log` inside the container, and streams audit records to a log stream named `kube-apiserver-audit-<hash>`.
+ - When both `api` and `audit` are enabled, both streams are created and populated under `/aws/eks/<cluster-name>/cluster`. When neither is enabled, no log group or streams are created.
+- **Component streams deviation**: AWS EKS provisions separate log streams for each component (`kube-apiserver-*`, `kube-apiserver-audit-*`, `kube-controller-manager-*`, `kube-scheduler-*`, `authenticator-*`). Because Floci runs clusters on k3s, which embeds the Kubernetes API server, controller manager, and scheduler within a single process, control plane container logs are delivered to the single `kube-apiserver-<hash>` stream when `api` is enabled, and API server audit logs are delivered to `kube-apiserver-audit-<hash>` when `audit` is enabled. Other control plane log types (`authenticator`, `controllerManager`, `scheduler`) do not provision separate streams.
+- **Authenticator logs**: Authenticator webhook authentication events are logged directly by Floci.
## Node group inputs
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
index 769f9cac16..7201c62e87 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
@@ -107,6 +107,14 @@ public class EksClusterManager {
static final String SA_PUBLIC_KEY_CONTAINER_PATH = WEBHOOK_CONFIG_DIR + "/" + SA_PUBLIC_KEY_FILE;
static final String KUBERNETES_DEFAULT_ISSUER = "https://kubernetes.default.svc.cluster.local";
+ static final String AUDIT_POLICY_FILE = "audit-policy.yaml";
+ static final String AUDIT_POLICY_DIR = "/etc";
+ static final String AUDIT_POLICY_CONTAINER_PATH = AUDIT_POLICY_DIR + "/" + AUDIT_POLICY_FILE;
+ static final String AUDIT_LOG_CONTAINER_PATH = "/var/log/audit.log";
+ static final String AUDIT_LOG_MAXAGE = "30";
+ static final String AUDIT_LOG_MAXBACKUP = "10";
+ static final String AUDIT_LOG_MAXSIZE = "100";
+
private final ContainerBuilder containerBuilder;
private final ContainerLifecycleManager lifecycleManager;
private final ContainerDetector containerDetector;
@@ -357,6 +365,18 @@ public class EksClusterManager {
}
}
+ String auditPolicyLocalFile = null;
+ if (hasLoggingEnabled(cluster, "audit")) {
+ auditPolicyLocalFile = writeAuditPolicyFile(cluster);
+ if (auditPolicyLocalFile != null) {
+ serverArgs.add("--kube-apiserver-arg=audit-policy-file=" + AUDIT_POLICY_CONTAINER_PATH);
+ serverArgs.add("--kube-apiserver-arg=audit-log-path=" + AUDIT_LOG_CONTAINER_PATH);
+ serverArgs.add("--kube-apiserver-arg=audit-log-maxage=" + AUDIT_LOG_MAXAGE);
+ serverArgs.add("--kube-apiserver-arg=audit-log-maxbackup=" + AUDIT_LOG_MAXBACKUP);
+ serverArgs.add("--kube-apiserver-arg=audit-log-maxsize=" + AUDIT_LOG_MAXSIZE);
+ }
+ }
+
if (config.services().eks().disableCni()) {
// A container's /sys mount defaults to private propagation, which breaks
// Cilium's BPF filesystem mount ("mounted on /sys but it is not a shared or
@@ -378,6 +398,9 @@ public class EksClusterManager {
if (webhookLocalFile != null) {
copyWebhookIntoContainer(containerId, webhookLocalFile, cluster.getName());
}
+ if (auditPolicyLocalFile != null) {
+ copyAuditPolicyIntoContainer(containerId, auditPolicyLocalFile, cluster.getName());
+ }
injectEcrRegistryMirror(containerId, cluster.getName());
registerPodIdentityWebhook(containerId, cluster);
if (signingKeyFiles != null) {
@@ -572,10 +595,10 @@ public class EksClusterManager {
}
/**
- * Checks whether the cluster has API server control plane logging enabled.
+ * Checks whether the cluster has control plane logging enabled for either api or audit.
*/
public static boolean hasLoggingEnabled(Cluster cluster) {
- return hasLoggingEnabled(cluster, "api");
+ return hasLoggingEnabled(cluster, "api") || hasLoggingEnabled(cluster, "audit");
}
/**
@@ -625,26 +648,77 @@ public class EksClusterManager {
if (clusterLogHandles.containsKey(resourceName)) {
return;
}
- try {
- String logGroup = "/aws/eks/" + cluster.getName() + "/cluster";
- String hash = containerId.length() >= 32 ? containerId.substring(0, 32) : containerId;
- String logStream = "kube-apiserver-" + hash;
- String region = clusterRegion(cluster);
- String accountId = resolveClusterAccountId(cluster);
- Closeable handle = fromNow
- ? logStreamer.attachFromNowForAccount(
- accountId, containerId, logGroup, logStream, region, "eks:" + cluster.getName())
- : logStreamer.attachForAccount(
- accountId, containerId, logGroup, logStream, region, "eks:" + cluster.getName());
- if (handle != null) {
- clusterLogHandles.put(resourceName, handle);
+ String logGroup = "/aws/eks/" + cluster.getName() + "/cluster";
+ String hash = containerId.length() >= 32 ? containerId.substring(0, 32) : containerId;
+ String region = clusterRegion(cluster);
+ String accountId = resolveClusterAccountId(cluster);
+
+ List<Closeable> handles = new ArrayList<>();
+ if (hasLoggingEnabled(cluster, "api")) {
+ try {
+ String logStream = "kube-apiserver-" + hash;
+ Closeable handle = fromNow
+ ? logStreamer.attachFromNowForAccount(
+ accountId, containerId, logGroup, logStream, region, "eks:" + cluster.getName())
+ : logStreamer.attachForAccount(
+ accountId, containerId, logGroup, logStream, region, "eks:" + cluster.getName());
+ if (handle != null) {
+ handles.add(handle);
+ }
+ } catch (Exception e) {
+ LOG.warnv("Could not attach control plane log stream for EKS cluster {0}: {1}",
+ cluster.getName(), e.getMessage());
+ }
+ }
+
+ if (hasLoggingEnabled(cluster, "audit")) {
+ try {
+ Closeable auditHandle = attachAuditLogFollower(
+ containerId, accountId, logGroup, hash, region, cluster.getName(), fromNow);
+ if (auditHandle != null) {
+ handles.add(auditHandle);
+ }
+ } catch (Exception e) {
+ LOG.warnv("Could not attach audit log follower for EKS cluster {0}: {1}",
+ cluster.getName(), e.getMessage());
}
- } catch (Exception e) {
- LOG.warnv("Could not attach control plane log stream for EKS cluster {0}: {1}",
- cluster.getName(), e.getMessage());
+ }
+
+ if (handles.size() == 1) {
+ clusterLogHandles.put(resourceName, handles.getFirst());
+ } else if (handles.size() > 1) {
+ clusterLogHandles.put(resourceName, () -> {
+ for (Closeable h : handles) {
+ closeQuietly(h);
+ }
+ });
}
}
+ private Closeable attachAuditLogFollower(String containerId, String accountId, String logGroup,
+ String hash, String region, String clusterName, boolean fromNow) {
+ DockerClient dockerClient = lifecycleManager.getDockerClient();
+ if (dockerClient == null) {
+ return null;
+ }
+ String logStream = "kube-apiserver-audit-" + hash;
+ logStreamer.ensureLogGroupAndStreamForAccount(accountId, logGroup, logStream, region);
+ String tailLineArg = fromNow ? "0" : "+1";
+ String[] cmd = new String[] {
+ "sh", "-c", "touch " + AUDIT_LOG_CONTAINER_PATH + " && exec tail -n " + tailLineArg + " -F " + AUDIT_LOG_CONTAINER_PATH
+ };
+ ExecCreateCmdResponse execCreate = dockerClient
+ .execCreateCmd(containerId)
+ .withCmd(cmd)
+ .withAttachStdout(true)
+ .withAttachStderr(false)
+ .exec();
+ return dockerClient
+ .execStartCmd(execCreate.getId())
+ .exec(logStreamer.execLogCallbackForAccount(
+ accountId, logGroup, logStream, region, "eks-audit:" + clusterName));
+ }
+
Closeable getLogHandle(Cluster cluster) {
return clusterLogHandles.get(clusterResourceName(cluster));
}
@@ -878,6 +952,233 @@ public class EksClusterManager {
}
}
+ /**
+ * Writes the official Amazon EKS audit policy YAML to Floci's local data directory and
+ * returns its path, or {@code null} if writing failed.
+ */
+ String writeAuditPolicyFile(Cluster cluster) {
+ String clusterName = cluster.getName();
+ try {
+ String dataPath = config.services().eks().dataPath();
+ if (dataPath == null || dataPath.isBlank()) {
+ return null;
+ }
+ Path localFile = Paths.get(dataPath, "audit", clusterName, AUDIT_POLICY_FILE)
+ .toAbsolutePath().normalize();
+ Files.createDirectories(localFile.getParent());
+ Files.writeString(localFile, buildAuditPolicy());
+ return localFile.toString();
+ } catch (Exception e) {
+ LOG.warnv("EKS audit logging disabled for cluster {0}: could not write audit policy file: {1}",
+ clusterName, e.getMessage());
+ return null;
+ }
+ }
+
+ /**
+ * Streams the audit policy from Floci's filesystem into the (created, not-yet-started)
+ * k3s container at {@value #AUDIT_POLICY_CONTAINER_PATH}, using the Docker API.
+ */
+ void copyAuditPolicyIntoContainer(String containerId, String localFile, String clusterName) {
+ try {
+ DockerClient dockerClient = lifecycleManager.getDockerClient();
+ if (dockerClient != null) {
+ dockerClient.copyArchiveToContainerCmd(containerId)
+ .withHostResource(localFile)
+ .withRemotePath(AUDIT_POLICY_DIR)
+ .exec();
+ LOG.debugv("Injected audit policy file into k3s container {0} for cluster {1}",
+ containerId, clusterName);
+ }
+ } catch (Exception e) {
+ LOG.warnv("EKS audit logs may not be emitted for cluster {0}: could not copy "
+ + "audit policy into the k3s container: {1}", clusterName, e.getMessage());
+ }
+ }
+
+ /**
+ * Official Amazon EKS control plane audit policy documented in the Amazon EKS Best Practices Guide.
+ */
+ public static String buildAuditPolicy() {
+ return """
+ apiVersion: audit.k8s.io/v1
+ kind: Policy
+ rules:
+ # Log full request and response for changes to aws-auth ConfigMap in kube-system namespace
+ - level: RequestResponse
+ namespaces: ["kube-system"]
+ verbs: ["update", "patch", "delete"]
+ resources:
+ - group: ""
+ resources: ["configmaps"]
+ resourceNames: ["aws-auth"]
+ omitStages:
+ - "RequestReceived"
+
+ # Do not log watch operations performed by kube-proxy on endpoints and services
+ - level: None
+ users: ["system:kube-proxy"]
+ verbs: ["watch"]
+ resources:
+ - group: ""
+ resources: ["endpoints", "services", "services/status"]
+
+ # Do not log get operations performed by kubelet on nodes and their statuses
+ - level: None
+ users: ["kubelet"]
+ verbs: ["get"]
+ resources:
+ - group: ""
+ resources: ["nodes", "nodes/status"]
+
+ # Do not log get operations performed by the system:nodes group on nodes and their statuses
+ - level: None
+ userGroups: ["system:nodes"]
+ verbs: ["get"]
+ resources:
+ - group: ""
+ resources: ["nodes", "nodes/status"]
+
+ # Do not log get and update operations performed by controller manager, scheduler, and endpoint-controller on endpoints in kube-system namespace
+ - level: None
+ users:
+ - system:kube-controller-manager
+ - system:kube-scheduler
+ - system:serviceaccount:kube-system:endpoint-controller
+ verbs: ["get", "update"]
+ namespaces: ["kube-system"]
+ resources:
+ - group: ""
+ resources: ["endpoints"]
+
+ # Do not log get operations performed by apiserver on namespaces and their statuses/finalizations
+ - level: None
+ users: ["system:apiserver"]
+ verbs: ["get"]
+ resources:
+ - group: ""
+ resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
+
+ # Do not log get and list operations performed by controller manager on metrics.k8s.io resources
+ - level: None
+ users:
+ - system:kube-controller-manager
+ verbs: ["get", "list"]
+ resources:
+ - group: "metrics.k8s.io"
+
+ # Do not log access to health, version, and swagger non-resource URLs
+ - level: None
+ nonResourceURLs:
+ - /healthz*
+ - /version
+ - /swagger*
+
+ # Do not log events resources
+ - level: None
+ resources:
+ - group: ""
+ resources: ["events"]
+
+ # Log request for updates/patches to nodes and pods statuses by kubelet and node problem detector
+ - level: Request
+ users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"]
+ verbs: ["update", "patch"]
+ resources:
+ - group: ""
+ resources: ["nodes/status", "pods/status"]
+ omitStages:
+ - "RequestReceived"
+
+ # Log request for updates/patches to nodes and pods statuses by system:nodes group
+ - level: Request
+ userGroups: ["system:nodes"]
+ verbs: ["update", "patch"]
+ resources:
+ - group: ""
+ resources: ["nodes/status", "pods/status"]
+ omitStages:
+ - "RequestReceived"
+
+ # Log delete collection requests by namespace-controller in kube-system namespace
+ - level: Request
+ users: ["system:serviceaccount:kube-system:namespace-controller"]
+ verbs: ["deletecollection"]
+ omitStages:
+ - "RequestReceived"
+
+ # Log metadata for secrets, configmaps, and tokenreviews to protect sensitive data
+ - level: Metadata
+ resources:
+ - group: ""
+ resources: ["secrets", "configmaps"]
+ - group: authentication.k8s.io
+ resources: ["tokenreviews"]
+ omitStages:
+ - "RequestReceived"
+
+ # Log requests for serviceaccounts/token resources
+ - level: Request
+ resources:
+ - group: ""
+ resources: ["serviceaccounts/token"]
+
+ # Log get, list, and watch requests for various resource groups
+ - level: Request
+ verbs: ["get", "list", "watch"]
+ resources:
+ - group: ""
+ - group: "admissionregistration.k8s.io"
+ - group: "apiextensions.k8s.io"
+ - group: "apiregistration.k8s.io"
+ - group: "apps"
+ - group: "authentication.k8s.io"
+ - group: "authorization.k8s.io"
+ - group: "autoscaling"
+ - group: "batch"
+ - group: "certificates.k8s.io"
+ - group: "extensions"
+ - group: "metrics.k8s.io"
+ - group: "networking.k8s.io"
+ - group: "policy"
+ - group: "rbac.authorization.k8s.io"
+ - group: "scheduling.k8s.io"
+ - group: "settings.k8s.io"
+ - group: "storage.k8s.io"
+ omitStages:
+ - "RequestReceived"
+
+ # Default logging level for known APIs to log request and response
+ - level: RequestResponse
+ resources:
+ - group: ""
+ - group: "admissionregistration.k8s.io"
+ - group: "apiextensions.k8s.io"
+ - group: "apiregistration.k8s.io"
+ - group: "apps"
+ - group: "authentication.k8s.io"
+ - group: "authorization.k8s.io"
+ - group: "autoscaling"
+ - group: "batch"
+ - group: "certificates.k8s.io"
+ - group: "extensions"
+ - group: "metrics.k8s.io"
+ - group: "networking.k8s.io"
+ - group: "policy"
+ - group: "rbac.authorization.k8s.io"
+ - group: "scheduling.k8s.io"
+ - group: "settings.k8s.io"
+ - group: "storage.k8s.io"
+ omitStages:
+ - "RequestReceived"
+
+ # Default logging level for all other requests to log metadata only
+ - level: Metadata
+ omitStages:
+ - "RequestReceived"
+ """;
+ }
+
/**
* Builds the API server arguments configuring k3s to sign service account tokens with the
* cluster's OIDC keypair and advertise the cluster's OIDC issuer URL. api-audiences includes
diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
index 60772e89df..91f3334257 100644
--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
@@ -73,6 +73,7 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
@@ -1653,6 +1654,7 @@ class EksClusterManagerTest {
private ContainerLogStreamer logStreamer;
private EksClusterManager manager;
private Closeable mockHandle;
+ private ContainerBuilder.Builder builder;
@BeforeEach
void setUp() {
@@ -1686,7 +1688,7 @@ class EksClusterManagerTest {
new ContainerInfo("container-id-123456789012345678901234567890", Map.of()));
ContainerBuilder containerBuilder = Mockito.mock(ContainerBuilder.class);
- ContainerBuilder.Builder builder = Mockito.mock(ContainerBuilder.Builder.class, Mockito.RETURNS_SELF);
+ builder = Mockito.mock(ContainerBuilder.Builder.class, Mockito.RETURNS_SELF);
when(containerBuilder.newContainer(anyString())).thenReturn(builder);
when(builder.build()).thenReturn(Mockito.mock(ContainerSpec.class));
@@ -1871,6 +1873,127 @@ class EksClusterManagerTest {
new LogSetup(List.of("scheduler"), false)
)));
assertTrue(EksClusterManager.hasLoggingEnabled(cluster));
+
+ cluster.setLogging(new Logging(List.of(
+ new LogSetup(List.of("audit"), true)
+ )));
+ assertTrue(EksClusterManager.hasLoggingEnabled(cluster));
+ assertTrue(EksClusterManager.hasLoggingEnabled(cluster, "audit"));
+ assertFalse(EksClusterManager.hasLoggingEnabled(cluster, "api"));
+ }
+
+ @Test
+ void clusterWithAuditLoggingAddsAuditArgsAndInjectsPolicyFile(@TempDir Path tempDir) {
+ when(eks.dataPath()).thenReturn(tempDir.toString());
+ DockerClient dockerClient = Mockito.mock(DockerClient.class);
+ CopyArchiveToContainerCmd copyCmd = Mockito.mock(CopyArchiveToContainerCmd.class, Mockito.RETURNS_SELF);
+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+ when(dockerClient.copyArchiveToContainerCmd(anyString())).thenReturn(copyCmd);
+
+ Cluster cluster = new Cluster();
+ cluster.setName("audit-cluster");
+ cluster.setLogging(new Logging(List.of(new LogSetup(List.of("audit"), true))));
+
+ manager.startCluster(cluster);
+
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor<List<String>> cmdCaptor = ArgumentCaptor.forClass(List.class);
+ verify(builder).withCmd(cmdCaptor.capture());
+ List<String> cmd = cmdCaptor.getValue();
+
+ assertTrue(cmd.contains("--kube-apiserver-arg=audit-policy-file=/etc/audit-policy.yaml"));
+ assertTrue(cmd.contains("--kube-apiserver-arg=audit-log-path=/var/log/audit.log"));
+ assertTrue(cmd.contains("--kube-apiserver-arg=audit-log-maxage=30"));
+ assertTrue(cmd.contains("--kube-apiserver-arg=audit-log-maxbackup=10"));
+ assertTrue(cmd.contains("--kube-apiserver-arg=audit-log-maxsize=100"));
+
+ verify(dockerClient).copyArchiveToContainerCmd("container-id-123456789012345678901234567890");
+ verify(copyCmd).withRemotePath("/etc");
+ }
+
+ @Test
+ void clusterWithoutAuditLoggingDoesNotAddAuditArgsOrInjectPolicyFile(@TempDir Path tempDir) {
+ when(eks.dataPath()).thenReturn(tempDir.toString());
+ DockerClient dockerClient = Mockito.mock(DockerClient.class);
+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+
+ Cluster cluster = new Cluster();
+ cluster.setName("no-audit-cluster");
+ cluster.setLogging(new Logging(List.of(new LogSetup(List.of("api"), true))));
+
+ manager.startCluster(cluster);
+
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor<List<String>> cmdCaptor = ArgumentCaptor.forClass(List.class);
+ verify(builder).withCmd(cmdCaptor.capture());
+ List<String> cmd = cmdCaptor.getValue();
+
+ assertFalse(cmd.stream().anyMatch(arg -> arg.contains("audit-policy-file")));
+ assertFalse(cmd.stream().anyMatch(arg -> arg.contains("audit-log-path")));
+
+ verify(dockerClient, never()).copyArchiveToContainerCmd(anyString());
+ }
+
+ @Test
+ void copyAuditPolicyFailureLogsWarningAndContinuesStartup(@TempDir Path tempDir) {
+ when(eks.dataPath()).thenReturn(tempDir.toString());
+ DockerClient dockerClient = Mockito.mock(DockerClient.class);
+ CopyArchiveToContainerCmd copyCmd = Mockito.mock(CopyArchiveToContainerCmd.class, Mockito.RETURNS_SELF);
+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+ when(dockerClient.copyArchiveToContainerCmd(anyString())).thenReturn(copyCmd);
+ doThrow(new RuntimeException("Docker copy failed")).when(copyCmd).exec();
+
+ Cluster cluster = new Cluster();
+ cluster.setName("audit-cluster");
+ cluster.setLogging(new Logging(List.of(new LogSetup(List.of("audit"), true))));
+
+ assertDoesNotThrow(() -> manager.startCluster(cluster));
+ assertEquals("container-id-123456789012345678901234567890", cluster.getContainerId());
+ }
+
+ @Test
+ void auditFollowerHandleIsReleasedOnClusterStop() throws Exception {
+ DockerClient dockerClient = Mockito.mock(DockerClient.class);
+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+ ExecCreateCmd execCreateCmd = Mockito.mock(ExecCreateCmd.class, Mockito.RETURNS_SELF);
+ ExecCreateCmdResponse execCreate = Mockito.mock(ExecCreateCmdResponse.class);
+ when(execCreate.getId()).thenReturn("exec-123");
+ when(dockerClient.execCreateCmd(anyString())).thenReturn(execCreateCmd);
+ when(execCreateCmd.exec()).thenReturn(execCreate);
+
+ ExecStartCmd execStartCmd = Mockito.mock(ExecStartCmd.class, Mockito.RETURNS_SELF);
+ when(dockerClient.execStartCmd("exec-123")).thenReturn(execStartCmd);
+ ResultCallback.Adapter<?> mockAuditHandle = Mockito.mock(ResultCallback.Adapter.class);
+ Mockito.doReturn(mockAuditHandle).when(execStartCmd).exec(any());
+
+ Cluster cluster = new Cluster();
+ cluster.setName("audit-cluster");
+ cluster.setLogging(new Logging(List.of(new LogSetup(List.of("audit"), true))));
+
+ manager.startCluster(cluster);
+ assertEquals(mockAuditHandle, manager.getLogHandle(cluster));
+
+ manager.stopCluster(cluster);
+ verify(lifecycleManager).stopAndRemove("container-id-123456789012345678901234567890", mockAuditHandle);
+ assertNull(manager.getLogHandle(cluster));
+ }
+
+ @Test
+ void auditPolicyMatchesExpectedAwsEksRules() {
+ String policy = EksClusterManager.buildAuditPolicy();
+ assertNotNull(policy);
+ assertTrue(policy.contains("apiVersion: audit.k8s.io/v1"));
+ assertTrue(policy.contains("kind: Policy"));
+ assertTrue(policy.contains("resourceNames: [\"aws-auth\"]"));
+ assertTrue(policy.contains("users: [\"system:kube-proxy\"]"));
+ assertTrue(policy.contains("userGroups: [\"system:nodes\"]"));
+ assertTrue(policy.contains("users: [\"kubelet\"]"));
+ assertTrue(policy.contains("resources: [\"secrets\", \"configmaps\"]"));
+ assertTrue(policy.contains("resources: [\"tokenreviews\"]"));
+ assertTrue(policy.contains("resources: [\"events\"]"));
+ assertTrue(policy.contains("nonResourceURLs:"));
+ assertTrue(policy.contains("/healthz*"));
+ assertTrue(policy.contains("/version"));
}
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksControlPlaneLogsDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksControlPlaneLogsDockerIntegrationTest.java
index 9623b92680..93e50fed65 100644
--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksControlPlaneLogsDockerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksControlPlaneLogsDockerIntegrationTest.java
@@ -239,6 +239,128 @@ class EksControlPlaneLogsDockerIntegrationTest {
assertTrue(groupsResult.isEmpty(), "Log group should not exist when logging is disabled");
}
+ @Test
+ void clusterWithAuditLoggingDeliversAuditEventsToCloudWatchLogs() throws Exception {
+ String clusterName = "audit-it-" + UUID.randomUUID().toString().substring(0, 8);
+ cluster = new Cluster();
+ cluster.setName(clusterName);
+ cluster.setAccountId(ACCOUNT);
+ cluster.setArn("arn:aws:eks:" + REGION + ":" + ACCOUNT + ":cluster/" + clusterName);
+ cluster.setLogging(new Logging(List.of(new LogSetup(List.of("audit"), true))));
+
+ String auditEventJson = "{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"stage\":\"ResponseComplete\",\"verb\":\"get\",\"user\":{\"username\":\"test-admin\"}}";
+ ContainerSpec spec = containerBuilder.newContainer(TEST_IMAGE)
+ .withName("floci-eks-cp-test-" + clusterName)
+ .withCmd(List.of("sh", "-c", "mkdir -p /var/log && echo '" + auditEventJson + "' >> /var/log/audit.log; sleep 30"))
+ .build();
+
+ containerId = lifecycleManager.createAndStart(spec).containerId();
+ assertNotNull(containerId, "Container ID must not be null");
+ cluster.setContainerId(containerId);
+
+ eksClusterManager.attachClusterLogs(cluster);
+ assertNotNull(eksClusterManager.getLogHandle(cluster), "Log handle should be active after attach");
+
+ String logGroup = "/aws/eks/" + clusterName + "/cluster";
+ String streamPrefix = "kube-apiserver-audit-";
+
+ List<LogStream> streams = List.of();
+ long deadline = System.currentTimeMillis() + 10_000;
+ while (System.currentTimeMillis() < deadline) {
+ try {
+ streams = cloudWatchLogsService.describeLogStreams(logGroup, streamPrefix, REGION);
+ if (!streams.isEmpty()) {
+ break;
+ }
+ } catch (Exception ignored) {
+ }
+ Thread.sleep(100);
+ }
+ assertFalse(streams.isEmpty(), "Expected log stream starting with " + streamPrefix + " in group " + logGroup);
+
+ String streamName = streams.get(0).getLogStreamName();
+ assertTrue(streamName.startsWith(streamPrefix), "Stream name should start with prefix: " + streamName);
+
+ List<LogEvent> events = List.of();
+ while (System.currentTimeMillis() < deadline) {
+ try {
+ LogEventsResult result = cloudWatchLogsService.getLogEvents(
+ logGroup, streamName, null, null, 10, true, null, REGION);
+ if (!result.events().isEmpty()) {
+ events = result.events();
+ break;
+ }
+ } catch (Exception ignored) {
+ }
+ Thread.sleep(100);
+ }
+ assertFalse(events.isEmpty(), "Expected audit log events in stream " + streamName);
+ assertTrue(events.stream().anyMatch(e -> e.getMessage().contains("\"kind\":\"Event\"")
+ && e.getMessage().contains("\"apiVersion\":\"audit.k8s.io/v1\"")),
+ "Expected audit event containing kind Event and apiVersion audit.k8s.io/v1");
+
+ eksClusterManager.stopCluster(cluster);
+ assertNull(eksClusterManager.getLogHandle(cluster), "Log handle should be removed after stop");
+ }
+
+ @Test
+ void clusterWithBothApiAndAuditLoggingDeliversToBothStreams() throws Exception {
+ String clusterName = "both-it-" + UUID.randomUUID().toString().substring(0, 8);
+ cluster = new Cluster();
+ cluster.setName(clusterName);
+ cluster.setAccountId(ACCOUNT);
+ cluster.setArn("arn:aws:eks:" + REGION + ":" + ACCOUNT + ":cluster/" + clusterName);
+ cluster.setLogging(new Logging(List.of(
+ new LogSetup(List.of("api", "audit"), true)
+ )));
+
+ String apiMsg = "api-msg-" + UUID.randomUUID();
+ String auditMsg = "{\"kind\":\"Event\",\"auditId\":\"" + UUID.randomUUID() + "\"}";
+
+ ContainerSpec spec = containerBuilder.newContainer(TEST_IMAGE)
+ .withName("floci-eks-cp-test-" + clusterName)
+ .withCmd(List.of("sh", "-c", "echo '" + apiMsg + "'; mkdir -p /var/log && echo '" + auditMsg + "' >> /var/log/audit.log; sleep 30"))
+ .build();
+
+ containerId = lifecycleManager.createAndStart(spec).containerId();
+ cluster.setContainerId(containerId);
+
+ eksClusterManager.attachClusterLogs(cluster);
+ assertNotNull(eksClusterManager.getLogHandle(cluster));
+
+ String logGroup = "/aws/eks/" + clusterName + "/cluster";
+ long deadline = System.currentTimeMillis() + 10_000;
+
+ List<LogStream> apiStreams = List.of();
+ while (System.currentTimeMillis() < deadline) {
+ try {
+ apiStreams = cloudWatchLogsService.describeLogStreams(logGroup, "kube-apiserver-", REGION);
+ if (!apiStreams.isEmpty()) {
+ break;
+ }
+ } catch (Exception ignored) {
+ }
+ Thread.sleep(100);
+ }
+ assertFalse(apiStreams.isEmpty(), "Expected kube-apiserver- stream");
+
+ List<LogStream> auditStreams = List.of();
+ while (System.currentTimeMillis() < deadline) {
+ try {
+ auditStreams = cloudWatchLogsService.describeLogStreams(logGroup, "kube-apiserver-audit-", REGION);
+ if (!auditStreams.isEmpty()) {
+ break;
+ }
+ } catch (Exception ignored) {
+ }
+ Thread.sleep(100);
+ }
+ assertFalse(auditStreams.isEmpty(), "Expected kube-apiserver-audit- stream");
+
+ eksClusterManager.stopCluster(cluster);
+ assertNull(eksClusterManager.getLogHandle(cluster));
+ }
+
private boolean isDockerAvailable() {
try {
dockerClient.pingCmd().exec();