Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(docker): add cpu limits and readonly rootfs to container specs (#3893)

Abhigya0f76840ef431
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.java
index 7d1abdb989..7b8eca2c3e 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.java
@@ -127,6 +127,9 @@ public class ContainerBuilder {
private List<String> entrypoint;
private String workingDir;
private Long memoryBytes;
+ private Long nanoCpus;
+ private Integer cpuShares;
+ private boolean readonlyRootfs;
private final Map<Integer, Integer> portBindings = new HashMap<>();
private final List<Integer> loopbackPortBindings = new ArrayList<>();
private final List<Integer> exposedPorts = new ArrayList<>();
@@ -226,6 +229,30 @@ public class ContainerBuilder {
return this;
}
+ /**
+ * Caps the container at a fraction of the host's CPUs, expressed the way ECS expresses it:
+ * 1024 CPU units is one vCPU.
+ */
+ public Builder withCpuUnits(int cpuUnits) {
+ this.nanoCpus = cpuUnits * 1_000_000_000L / 1024L;
+ return this;
+ }
+
+ /**
+ * Sets the container's relative CPU weight, which is what a container-level {@code cpu}
+ * means when several containers share a task's CPU allocation.
+ */
+ public Builder withCpuShares(int cpuShares) {
+ this.cpuShares = cpuShares;
+ return this;
+ }
+
+ /** Mounts the container's own filesystem read only. */
+ public Builder withReadonlyRootfs() {
+ this.readonlyRootfs = true;
+ return this;
+ }
+
/**
* Adds a port binding from container port to a specific host port.
*/
@@ -612,7 +639,10 @@ public class ContainerBuilder {
workingDir,
user,
List.copyOf(groupAdd),
- List.copyOf(deviceRequests)
+ List.copyOf(deviceRequests),
+ nanoCpus,
+ cpuShares,
+ readonlyRootfs
);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index 20e5ca7dab..aab38c073e 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -40,6 +40,7 @@ import java.util.OptionalInt;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Pattern;
/**
@@ -58,6 +59,8 @@ public class ContainerLifecycleManager {
private static final Pattern KEYRING_QUOTA_PATTERN =
Pattern.compile("join keyctl.*disk quota exceeded", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
+ private static final long NANO_CPUS_PER_CPU = 1_000_000_000L;
+
private final DockerClient dockerClient;
private final ImageCacheService imageCacheService;
private final ContainerDetector containerDetector;
@@ -67,6 +70,9 @@ public class ContainerLifecycleManager {
/** Volumes whose shared-ownership root has already been initialised this process (run-once guard). */
private final ConcurrentHashMap<String, Boolean> initializedSharedVolumes = new ConcurrentHashMap<>();
+ /** Daemon CPU count, resolved on first use. Null until then, so that empty can mean "unknown". */
+ private final AtomicReference<OptionalInt> hostCpuCount = new AtomicReference<>();
+
@Inject
public ContainerLifecycleManager(DockerClient dockerClient,
ImageCacheService imageCacheService,
@@ -910,6 +916,17 @@ public class ContainerLifecycleManager {
hostConfig.withMemory(spec.memoryBytes());
}
+ // CPU quota and weight
+ if (spec.nanoCpus() != null && spec.nanoCpus() > 0) {
+ hostConfig.withNanoCPUs(clampToHostCpus(spec.nanoCpus()));
+ }
+ if (spec.cpuShares() != null && spec.cpuShares() > 0) {
+ hostConfig.withCpuShares(spec.cpuShares());
+ }
+ if (spec.readonlyRootfs()) {
+ hostConfig.withReadonlyRootfs(true);
+ }
+
// Port bindings — services decide whether to request them based on their
// own in-container-vs-native logic. When Floci runs inside Docker, most
// backends are reached via the docker network IP, so services omit the
@@ -986,6 +1003,46 @@ public class ContainerLifecycleManager {
return hostConfig;
}
+ /**
+ * Holds a CPU quota to what the daemon will accept. Docker rejects a NanoCpus above the
+ * host's CPU count ("range of CPUs is from 0.01 to 8.00, as there are only 8 CPUs
+ * available"), while ECS accepts a task cpu of up to 196608 units, so a 16 vCPU task
+ * asked for verbatim would simply fail to start on a smaller machine. Emulating the
+ * larger task on fewer CPUs is closer to the caller's intent than refusing to run it.
+ */
+ private long clampToHostCpus(long nanoCpus) {
+ OptionalInt cpus = hostCpuCount();
+ if (cpus.isEmpty()) {
+ return nanoCpus;
+ }
+ long ceiling = cpus.getAsInt() * NANO_CPUS_PER_CPU;
+ if (nanoCpus <= ceiling) {
+ return nanoCpus;
+ }
+ LOG.warnv("Requested {0} nanoCPUs but the Docker daemon reports {1} CPUs; clamping to {2}",
+ nanoCpus, cpus.getAsInt(), ceiling);
+ return ceiling;
+ }
+
+ /** The daemon's CPU count, resolved once. Empty when the daemon did not report one. */
+ private OptionalInt hostCpuCount() {
+ OptionalInt cached = hostCpuCount.get();
+ if (cached != null) {
+ return cached;
+ }
+ OptionalInt resolved = OptionalInt.empty();
+ try {
+ Integer ncpu = dockerClient.infoCmd().exec().getNCPU();
+ if (ncpu != null && ncpu > 0) {
+ resolved = OptionalInt.of(ncpu);
+ }
+ } catch (RuntimeException e) {
+ LOG.debugv(e, "Could not read the Docker daemon's CPU count; leaving CPU quotas unclamped");
+ }
+ hostCpuCount.set(resolved);
+ return resolved;
+ }
+
private static boolean isContainerNetworkMode(String networkMode) {
return networkMode != null && networkMode.startsWith("container:");
}
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerSpec.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerSpec.java
index 7e59fc2846..05a9ae0ded 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerSpec.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerSpec.java
@@ -36,6 +36,9 @@ import java.util.Map;
* @param user User the container process runs as, formatted "uid[:gid]" (null = image USER)
* @param groupAdd Supplementary group IDs added to the container process
* @param deviceRequests Device requests for accelerators such as GPUs (empty = none)
+ * @param nanoCpus Hard CPU quota in billionths of a CPU (null = no quota)
+ * @param cpuShares Relative CPU weight against other containers (null = daemon default)
+ * @param readonlyRootfs Whether the container's own filesystem is mounted read only
*/
public record ContainerSpec(
String image,
@@ -60,7 +63,10 @@ public record ContainerSpec(
String workingDir,
String user,
List<String> groupAdd,
- List<DeviceRequest> deviceRequests
+ List<DeviceRequest> deviceRequests,
+ Long nanoCpus,
+ Integer cpuShares,
+ boolean readonlyRootfs
) {
/**
* Creates a minimal spec with just the image name.
@@ -69,7 +75,7 @@ public record ContainerSpec(
public ContainerSpec(String image) {
this(image, null, List.of(), null, null, null, Map.of(), List.of(), List.of(), null,
List.of(), List.of(), List.of(), List.of(), Map.of(), null, false, null, List.of(),
- null, null, List.of(), List.of());
+ null, null, List.of(), List.of(), null, null, false);
}
/**
@@ -100,7 +106,7 @@ public record ContainerSpec(
) {
this(image, name, env, cmd, entrypoint, memoryBytes, portBindings, List.of(), exposedPorts,
networkMode, mounts, binds, List.of(), extraHosts, labels, logConfig, privileged,
- cgroupnsMode, dnsServers, workingDir, user, groupAdd, List.of());
+ cgroupnsMode, dnsServers, workingDir, user, groupAdd, List.of(), null, null, false);
}
/**
@@ -133,7 +139,8 @@ public record ContainerSpec(
) {
this(image, name, env, cmd, entrypoint, memoryBytes, portBindings, loopbackPortBindings,
exposedPorts, networkMode, mounts, binds, List.of(), extraHosts, labels, logConfig,
- privileged, cgroupnsMode, dnsServers, workingDir, user, groupAdd, List.of());
+ privileged, cgroupnsMode, dnsServers, workingDir, user, groupAdd, List.of(),
+ null, null, false);
}
/**
@@ -166,7 +173,8 @@ public record ContainerSpec(
) {
this(image, name, env, cmd, entrypoint, memoryBytes, portBindings, loopbackPortBindings,
exposedPorts, networkMode, mounts, binds, List.of(), extraHosts, labels, logConfig,
- privileged, cgroupnsMode, dnsServers, workingDir, user, groupAdd, deviceRequests);
+ privileged, cgroupnsMode, dnsServers, workingDir, user, groupAdd, deviceRequests,
+ null, null, false);
}
/**
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCpuLimitTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCpuLimitTest.java
new file mode 100644
index 0000000000..51d837c31a
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerCpuLimitTest.java
@@ -0,0 +1,150 @@
+package io.github.hectorvent.floci.core.common.docker;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CreateContainerCmd;
+import com.github.dockerjava.api.command.CreateContainerResponse;
+import com.github.dockerjava.api.command.InfoCmd;
+import com.github.dockerjava.api.model.HostConfig;
+import com.github.dockerjava.api.model.Info;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.util.Optional;
+import java.util.function.UnaryOperator;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.RETURNS_SELF;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Pins the CPU mapping a container spec carries into Docker's HostConfig: ECS CPU units
+ * become a nanoCPU quota, a container-level cpu stays a relative weight, and a quota larger
+ * than the host is held to what the daemon will accept rather than failing the launch.
+ */
+class ContainerLifecycleManagerCpuLimitTest {
+
+ private static final int EIGHT_CPU_HOST = 8;
+
+ @Test
+ void cpuUnitsBecomeANanoCpuQuota() {
+ HostConfig hostConfig = createHostConfig(EIGHT_CPU_HOST,
+ builder -> builder.withCpuUnits(512));
+
+ assertEquals(500_000_000L, hostConfig.getNanoCPUs());
+ }
+
+ @Test
+ void zeroCpuUnitsLeaveTheQuotaOff() {
+ HostConfig hostConfig = createHostConfig(EIGHT_CPU_HOST,
+ builder -> builder.withCpuUnits(0));
+
+ assertNull(hostConfig.getNanoCPUs());
+ }
+
+ @Test
+ void negativeCpuUnitsLeaveTheQuotaOff() {
+ HostConfig hostConfig = createHostConfig(EIGHT_CPU_HOST,
+ builder -> builder.withCpuUnits(-512));
+
+ assertNull(hostConfig.getNanoCPUs());
+ }
+
+ @Test
+ void cpuSharesPassThroughAsARelativeWeight() {
+ HostConfig hostConfig = createHostConfig(EIGHT_CPU_HOST,
+ builder -> builder.withCpuShares(256));
+
+ assertEquals(256, hostConfig.getCpuShares());
+ assertNull(hostConfig.getNanoCPUs());
+ }
+
+ @Test
+ void readonlyRootfsReachesHostConfig() {
+ HostConfig hostConfig = createHostConfig(EIGHT_CPU_HOST,
+ ContainerBuilder.Builder::withReadonlyRootfs);
+
+ assertTrue(hostConfig.getReadonlyRootfs());
+ }
+
+ @Test
+ void containerThatAskedForNoCpuLimitGetsNone() {
+ HostConfig hostConfig = createHostConfig(EIGHT_CPU_HOST, UnaryOperator.identity());
+
+ assertNull(hostConfig.getNanoCPUs());
+ assertNull(hostConfig.getCpuShares());
+ assertNull(hostConfig.getReadonlyRootfs());
+ }
+
+ /**
+ * A 16 vCPU Fargate task is a legal ECS size but Docker refuses a quota above the host's
+ * CPU count, so asking for it verbatim would fail the launch on any smaller machine.
+ */
+ @Test
+ void quotaAboveTheHostCpuCountIsClampedToTheDaemonsCpus() {
+ HostConfig hostConfig = createHostConfig(EIGHT_CPU_HOST,
+ builder -> builder.withCpuUnits(16 * 1024));
+
+ assertEquals(8_000_000_000L, hostConfig.getNanoCPUs());
+ }
+
+ @Test
+ void quotaAtTheHostCpuCountIsNotClamped() {
+ HostConfig hostConfig = createHostConfig(EIGHT_CPU_HOST,
+ builder -> builder.withCpuUnits(8 * 1024));
+
+ assertEquals(8_000_000_000L, hostConfig.getNanoCPUs());
+ }
+
+ /** A daemon that does not report a CPU count is no reason to rewrite the caller's quota. */
+ @Test
+ void quotaIsLeftAloneWhenTheDaemonReportsNoCpuCount() {
+ HostConfig hostConfig = createHostConfig(null,
+ builder -> builder.withCpuUnits(16 * 1024));
+
+ assertEquals(16_000_000_000L, hostConfig.getNanoCPUs());
+ }
+
+ private static HostConfig createHostConfig(Integer hostCpus,
+ UnaryOperator<ContainerBuilder.Builder> customizer) {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.docker().resourceNamespace()).thenReturn(Optional.empty());
+ ContainerBuilder builder = new ContainerBuilder(
+ config, mock(DockerHostResolver.class), mock(EmbeddedDnsServer.class));
+ ContainerSpec spec = customizer.apply(builder.newContainer("app:latest")).build();
+
+ DockerClient dockerClient = mock(DockerClient.class);
+ stubDaemonCpuCount(dockerClient, hostCpus);
+ CreateContainerCmd createCmd = mock(CreateContainerCmd.class, RETURNS_SELF);
+ when(dockerClient.createContainerCmd("app:latest")).thenReturn(createCmd);
+ CreateContainerResponse response = mock(CreateContainerResponse.class);
+ when(response.getId()).thenReturn("app-id");
+ when(createCmd.exec()).thenReturn(response);
+ ImageCacheService imageCacheService = mock(ImageCacheService.class);
+ when(imageCacheService.ensureImageExists(any())).thenAnswer(invocation -> invocation.getArgument(0));
+ ContainerLifecycleManager manager = new ContainerLifecycleManager(
+ dockerClient, imageCacheService, mock(ContainerDetector.class), mock(PortAllocator.class), config);
+
+ manager.create(spec);
+
+ ArgumentCaptor<HostConfig> hostConfig = ArgumentCaptor.forClass(HostConfig.class);
+ verify(createCmd).withHostConfig(hostConfig.capture());
+ return hostConfig.getValue();
+ }
+
+ private static void stubDaemonCpuCount(DockerClient dockerClient, Integer hostCpus) {
+ InfoCmd infoCmd = mock(InfoCmd.class);
+ Info info = mock(Info.class);
+ when(dockerClient.infoCmd()).thenReturn(infoCmd);
+ when(infoCmd.exec()).thenReturn(info);
+ when(info.getNCPU()).thenReturn(hostCpus);
+ }
+}