Light, fluffy, and always free - The AWS Local Emulator alternative
fix(docker): translate misleading disk-quota error for keyring exhaustion (#2797)
Olamide Kosile7828ab2430ba
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javaindex f1b9a63c6b..da2b316f27 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -32,6 +32,7 @@ import java.util.OptionalInt;import java.util.Set;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.TimeUnit;+import java.util.regex.Pattern;/*** Manages Docker container lifecycle operations including create, start, stop, and remove.@@ -42,6 +43,13 @@ public class ContainerLifecycleManager {private static final Logger LOG = Logger.getLogger(ContainerLifecycleManager.class);+ // Matches crun/runc's "join keyctl '<id>': Disk quota exceeded" error, which the OCI runtime+ // reports for the kernel session-keyring quota (kernel.keys.maxkeys), not actual disk space.+ // Under high concurrent container counts (podman machine's Fedora CoreOS default is 200 keys+ // per uid) this reads as a disk-space problem and sends operators looking in the wrong place.+ private static final Pattern KEYRING_QUOTA_PATTERN =+ Pattern.compile("join keyctl.*disk quota exceeded", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);+private final DockerClient dockerClient;private final ImageCacheService imageCacheService;private final ContainerDetector containerDetector;@@ -143,7 +151,7 @@ public class ContainerLifecycleManager {* @return information about the running container including resolved endpoints*/public ContainerInfo startCreated(String containerId, ContainerSpec spec) {- dockerClient.startContainerCmd(containerId).exec();+ startContainer(containerId);LOG.infov("Started container {0}", containerId);if (spec.networkMode() != null && !spec.networkMode().isBlank() && spec.hasPortBindings()) {@@ -163,6 +171,30 @@ public class ContainerLifecycleManager {return new ContainerInfo(containerId, endpoints);}+ /**+ * Starts a container, translating crun/runc's kernel-keyring-quota error into a message that+ * names the actual cause and a fix, instead of the misleading "Disk quota exceeded" the OCI+ * runtime reports. Package-private so tests can verify a given call site routes through this+ * translation rather than a raw {@code dockerClient.startContainerCmd(...)} call.+ */+ void startContainer(String containerId) {+ try {+ dockerClient.startContainerCmd(containerId).exec();+ } catch (DockerException e) {+ String message = e.getMessage();+ if (message != null && KEYRING_QUOTA_PATTERN.matcher(message).find()) {+ throw new IllegalStateException(+ "Container start failed: the container runtime's kernel session-keyring is "+ + "out of quota (kernel.keys.maxkeys), not disk space. This is common "+ + "under many concurrent containers on podman machine's default Fedora "+ + "CoreOS sysctls (200 keys per uid). Raise the limit in the VM, e.g.: "+ + "podman machine ssh \"sudo sysctl -w kernel.keys.maxkeys=20000 "+ + "kernel.keys.maxbytes=2000000\". Original error: " + message, e);+ }+ throw e;+ }+ }+/*** Stops and removes a container, closing any associated log stream.*@@ -381,7 +413,7 @@ public class ContainerLifecycleManager {.exec();String helperId = created.getId();try {- dockerClient.startContainerCmd(helperId).exec();+ startContainer(helperId);Integer status = dockerClient.waitContainerCmd(helperId).exec(new WaitContainerResultCallback()).awaitStatusCode(60, TimeUnit.SECONDS);@@ -540,7 +572,7 @@ public class ContainerLifecycleManager {boolean running = Boolean.TRUE.equals(inspect.getState().getRunning());if (!running) {- dockerClient.startContainerCmd(containerId).exec();+ startContainer(containerId);LOG.infov("Started adopted container {0}", containerId);inspect = dockerClient.inspectContainerCmd(containerId).exec();}diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerStartFailureTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerStartFailureTest.javaindex fe47c5d73b..3ddc49cdf0 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerStartFailureTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerStartFailureTest.java@@ -2,6 +2,8 @@ package io.github.hectorvent.floci.core.common.docker;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.command.RemoveContainerCmd;+import com.github.dockerjava.api.command.StartContainerCmd;+import com.github.dockerjava.api.exception.DockerException;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;import org.junit.jupiter.api.BeforeEach;@@ -14,6 +16,7 @@ import java.util.Map;import static org.junit.jupiter.api.Assertions.assertSame;import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.Mockito.doReturn;import static org.mockito.Mockito.doThrow;@@ -85,4 +88,37 @@ class ContainerLifecycleManagerStartFailureTest {verify(dockerClient, never()).removeContainerCmd(any(String.class));}++ @Test+ void startCreatedTranslatesKeyringQuotaError() {+ // github.com/floci-io/floci/issues/2243: crun/runc report the kernel session-keyring+ // quota (kernel.keys.maxkeys) as "Disk quota exceeded", which sends operators looking at+ // disk space instead of the actual cause under high concurrent container counts.+ StartContainerCmd startCmd = mock(StartContainerCmd.class);+ when(dockerClient.startContainerCmd("container-id")).thenReturn(startCmd);+ when(startCmd.exec()).thenThrow(new DockerException(+ "crun: join keyctl '12345': Disk quota exceeded: OCI runtime error", 500));++ IllegalStateException thrown = assertThrows(IllegalStateException.class,+ () -> manager.startCreated("container-id", spec));++ assertTrue(thrown.getMessage().contains("kernel.keys.maxkeys"),+ "translated message should name the actual kernel-keyring cause: " + thrown.getMessage());+ assertTrue(thrown.getMessage().contains("sysctl"),+ "translated message should include the fix: " + thrown.getMessage());+ }++ @Test+ void startCreatedPropagatesUnrelatedDockerExceptionsUnchanged() {+ StartContainerCmd startCmd = mock(StartContainerCmd.class);+ when(dockerClient.startContainerCmd("container-id")).thenReturn(startCmd);+ DockerException portConflict = new DockerException(+ "Bind for 0.0.0.0:80 failed: port is already allocated", 500);+ when(startCmd.exec()).thenThrow(portConflict);++ DockerException thrown = assertThrows(DockerException.class,+ () -> manager.startCreated("container-id", spec));++ assertSame(portConflict, thrown, "unrelated Docker errors must propagate unchanged");+ }}diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.javaindex d5ca2dd1ce..2f4944117f 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java@@ -388,4 +388,41 @@ class ContainerLifecycleManagerVolumeTest {verify(dockerClient, times(2)).createContainerCmd("busybox:stable");}++ @Test+ void ensureSharedVolume_startsHelperThroughTheTranslatingStartContainer() {+ // github.com/floci-io/floci/issues/2243 (follow-up, PR #2797 review): the shared-volume+ // init helper is a third container-start call site, distinct from createAndStart /+ // startCreated and from adopt(). A failure here is caught as a plain RuntimeException by+ // ensureSharedVolume's computeIfAbsent and only logged (never rethrown to the caller), so+ // whether the logged message is the misleading raw "Disk quota exceeded" or the translated+ // one depends entirely on whether this call site goes through startContainer(). Verifying+ // startContainer("helper-id") is actually invoked (via a spy) is the only way to pin that+ // routing, since the swallowed exception itself is not observable from ensureSharedVolume.+ ContainerLifecycleManager spyManager = spy(manager);+ doNothing().when(spyManager).startContainer("helper-id");++ InspectVolumeCmd ivc = mock(InspectVolumeCmd.class);+ when(dockerClient.inspectVolumeCmd("shared")).thenReturn(ivc);+ when(ivc.exec()).thenReturn(mock(InspectVolumeResponse.class));++ CreateContainerCmd ccc = mock(CreateContainerCmd.class, RETURNS_SELF);+ when(dockerClient.createContainerCmd("busybox:stable")).thenReturn(ccc);+ CreateContainerResponse resp = mock(CreateContainerResponse.class);+ when(resp.getId()).thenReturn("helper-id");+ when(ccc.exec()).thenReturn(resp);++ WaitContainerCmd wcc = mock(WaitContainerCmd.class);+ when(dockerClient.waitContainerCmd("helper-id")).thenReturn(wcc);+ WaitContainerResultCallback wcb = mock(WaitContainerResultCallback.class);+ when(wcc.exec(any(WaitContainerResultCallback.class))).thenReturn(wcb);+ when(wcb.awaitStatusCode(anyLong(), any())).thenReturn(0);+ when(dockerClient.removeContainerCmd("helper-id")).thenReturn(mock(RemoveContainerCmd.class, RETURNS_SELF));++ spyManager.ensureSharedVolume("shared", OptionalInt.of(1001), OptionalInt.of(1001),+ Optional.of("2775"), "busybox:stable");++ verify(spyManager).startContainer("helper-id");+ verify(dockerClient, never()).startContainerCmd("helper-id");+ }}