Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(docker): translate misleading disk-quota error for keyring exhaustion (#2797)

Olamide Kosile7828ab2430ba
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index f1b9a63c6b..da2b316f27 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -32,6 +32,7 @@ import java.util.OptionalInt;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
/**
* Manages Docker container lifecycle operations including create, start, stop, and remove.
@@ -42,6 +43,13 @@ public class ContainerLifecycleManager {
private static final Logger LOG = Logger.getLogger(ContainerLifecycleManager.class);
+ // Matches crun/runc's "join keyctl '<id>': Disk quota exceeded" error, which the OCI runtime
+ // reports for the kernel session-keyring quota (kernel.keys.maxkeys), not actual disk space.
+ // Under high concurrent container counts (podman machine's Fedora CoreOS default is 200 keys
+ // per uid) this reads as a disk-space problem and sends operators looking in the wrong place.
+ private static final Pattern KEYRING_QUOTA_PATTERN =
+ Pattern.compile("join keyctl.*disk quota exceeded", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
+
private final DockerClient dockerClient;
private final ImageCacheService imageCacheService;
private final ContainerDetector containerDetector;
@@ -143,7 +151,7 @@ public class ContainerLifecycleManager {
* @return information about the running container including resolved endpoints
*/
public ContainerInfo startCreated(String containerId, ContainerSpec spec) {
- dockerClient.startContainerCmd(containerId).exec();
+ startContainer(containerId);
LOG.infov("Started container {0}", containerId);
if (spec.networkMode() != null && !spec.networkMode().isBlank() && spec.hasPortBindings()) {
@@ -163,6 +171,30 @@ public class ContainerLifecycleManager {
return new ContainerInfo(containerId, endpoints);
}
+ /**
+ * Starts a container, translating crun/runc's kernel-keyring-quota error into a message that
+ * names the actual cause and a fix, instead of the misleading "Disk quota exceeded" the OCI
+ * runtime reports. Package-private so tests can verify a given call site routes through this
+ * translation rather than a raw {@code dockerClient.startContainerCmd(...)} call.
+ */
+ void startContainer(String containerId) {
+ try {
+ dockerClient.startContainerCmd(containerId).exec();
+ } catch (DockerException e) {
+ String message = e.getMessage();
+ if (message != null && KEYRING_QUOTA_PATTERN.matcher(message).find()) {
+ throw new IllegalStateException(
+ "Container start failed: the container runtime's kernel session-keyring is "
+ + "out of quota (kernel.keys.maxkeys), not disk space. This is common "
+ + "under many concurrent containers on podman machine's default Fedora "
+ + "CoreOS sysctls (200 keys per uid). Raise the limit in the VM, e.g.: "
+ + "podman machine ssh \"sudo sysctl -w kernel.keys.maxkeys=20000 "
+ + "kernel.keys.maxbytes=2000000\". Original error: " + message, e);
+ }
+ throw e;
+ }
+ }
+
/**
* Stops and removes a container, closing any associated log stream.
*
@@ -381,7 +413,7 @@ public class ContainerLifecycleManager {
.exec();
String helperId = created.getId();
try {
- dockerClient.startContainerCmd(helperId).exec();
+ startContainer(helperId);
Integer status = dockerClient.waitContainerCmd(helperId)
.exec(new WaitContainerResultCallback())
.awaitStatusCode(60, TimeUnit.SECONDS);
@@ -540,7 +572,7 @@ public class ContainerLifecycleManager {
boolean running = Boolean.TRUE.equals(inspect.getState().getRunning());
if (!running) {
- dockerClient.startContainerCmd(containerId).exec();
+ startContainer(containerId);
LOG.infov("Started adopted container {0}", containerId);
inspect = dockerClient.inspectContainerCmd(containerId).exec();
}
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerStartFailureTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerStartFailureTest.java
index fe47c5d73b..3ddc49cdf0 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerStartFailureTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerStartFailureTest.java
@@ -2,6 +2,8 @@ package io.github.hectorvent.floci.core.common.docker;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.RemoveContainerCmd;
+import com.github.dockerjava.api.command.StartContainerCmd;
+import com.github.dockerjava.api.exception.DockerException;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;
import org.junit.jupiter.api.BeforeEach;
@@ -14,6 +16,7 @@ import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
@@ -85,4 +88,37 @@ class ContainerLifecycleManagerStartFailureTest {
verify(dockerClient, never()).removeContainerCmd(any(String.class));
}
+
+ @Test
+ void startCreatedTranslatesKeyringQuotaError() {
+ // github.com/floci-io/floci/issues/2243: crun/runc report the kernel session-keyring
+ // quota (kernel.keys.maxkeys) as "Disk quota exceeded", which sends operators looking at
+ // disk space instead of the actual cause under high concurrent container counts.
+ StartContainerCmd startCmd = mock(StartContainerCmd.class);
+ when(dockerClient.startContainerCmd("container-id")).thenReturn(startCmd);
+ when(startCmd.exec()).thenThrow(new DockerException(
+ "crun: join keyctl '12345': Disk quota exceeded: OCI runtime error", 500));
+
+ IllegalStateException thrown = assertThrows(IllegalStateException.class,
+ () -> manager.startCreated("container-id", spec));
+
+ assertTrue(thrown.getMessage().contains("kernel.keys.maxkeys"),
+ "translated message should name the actual kernel-keyring cause: " + thrown.getMessage());
+ assertTrue(thrown.getMessage().contains("sysctl"),
+ "translated message should include the fix: " + thrown.getMessage());
+ }
+
+ @Test
+ void startCreatedPropagatesUnrelatedDockerExceptionsUnchanged() {
+ StartContainerCmd startCmd = mock(StartContainerCmd.class);
+ when(dockerClient.startContainerCmd("container-id")).thenReturn(startCmd);
+ DockerException portConflict = new DockerException(
+ "Bind for 0.0.0.0:80 failed: port is already allocated", 500);
+ when(startCmd.exec()).thenThrow(portConflict);
+
+ DockerException thrown = assertThrows(DockerException.class,
+ () -> manager.startCreated("container-id", spec));
+
+ assertSame(portConflict, thrown, "unrelated Docker errors must propagate unchanged");
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
index d5ca2dd1ce..2f4944117f 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
@@ -388,4 +388,41 @@ class ContainerLifecycleManagerVolumeTest {
verify(dockerClient, times(2)).createContainerCmd("busybox:stable");
}
+
+ @Test
+ void ensureSharedVolume_startsHelperThroughTheTranslatingStartContainer() {
+ // github.com/floci-io/floci/issues/2243 (follow-up, PR #2797 review): the shared-volume
+ // init helper is a third container-start call site, distinct from createAndStart /
+ // startCreated and from adopt(). A failure here is caught as a plain RuntimeException by
+ // ensureSharedVolume's computeIfAbsent and only logged (never rethrown to the caller), so
+ // whether the logged message is the misleading raw "Disk quota exceeded" or the translated
+ // one depends entirely on whether this call site goes through startContainer(). Verifying
+ // startContainer("helper-id") is actually invoked (via a spy) is the only way to pin that
+ // routing, since the swallowed exception itself is not observable from ensureSharedVolume.
+ ContainerLifecycleManager spyManager = spy(manager);
+ doNothing().when(spyManager).startContainer("helper-id");
+
+ InspectVolumeCmd ivc = mock(InspectVolumeCmd.class);
+ when(dockerClient.inspectVolumeCmd("shared")).thenReturn(ivc);
+ when(ivc.exec()).thenReturn(mock(InspectVolumeResponse.class));
+
+ CreateContainerCmd ccc = mock(CreateContainerCmd.class, RETURNS_SELF);
+ when(dockerClient.createContainerCmd("busybox:stable")).thenReturn(ccc);
+ CreateContainerResponse resp = mock(CreateContainerResponse.class);
+ when(resp.getId()).thenReturn("helper-id");
+ when(ccc.exec()).thenReturn(resp);
+
+ WaitContainerCmd wcc = mock(WaitContainerCmd.class);
+ when(dockerClient.waitContainerCmd("helper-id")).thenReturn(wcc);
+ WaitContainerResultCallback wcb = mock(WaitContainerResultCallback.class);
+ when(wcc.exec(any(WaitContainerResultCallback.class))).thenReturn(wcb);
+ when(wcb.awaitStatusCode(anyLong(), any())).thenReturn(0);
+ when(dockerClient.removeContainerCmd("helper-id")).thenReturn(mock(RemoveContainerCmd.class, RETURNS_SELF));
+
+ spyManager.ensureSharedVolume("shared", OptionalInt.of(1001), OptionalInt.of(1001),
+ Optional.of("2775"), "busybox:stable");
+
+ verify(spyManager).startContainer("helper-id");
+ verify(dockerClient, never()).startContainerCmd("helper-id");
+ }
}