Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ecs): bound FireLens socket volume paths (#6787)
Andre Kurait2d3fb0ce8764
diff --git a/docs/services/ecs.md b/docs/services/ecs.mdindex c222118a11..c0df41e151 100644--- a/docs/services/ecs.md+++ b/docs/services/ecs.md@@ -196,6 +196,12 @@ other case, the injected `FLUENT_HOST` (the router's container IP) must be reachconfig is written to `/fluent-bit/etc/fluent-bit.conf`. Fluentd config iswritten to `/fluentd/etc/fluent.conf` and uses `@type` (not `Name`) for output plugins; Flocidoes not inject an `endpoint` into Fluentd outputs.+FireLens uses a bounded socket-volume name derived from the complete deployment and task identity,+while its Docker labels retain the full configured resource namespace. The inspected host socket+path must fit Linux's 107-byte Unix socket pathname limit. An unusually long Docker data root can+still exceed that limit; launch then fails before creating containers and retains any failed volume+cleanup for retry. Operator-named volumes are unaffected.+FireLens configuration archive uploads retry transient Docker connection failures up to sixattempts, with exponential backoff starting at 500 ms. Each attempt sends the complete prebuiltarchive through a fresh byte-backed stream, without a streaming pipe or writer thread.diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.javaindex 4bc5d1111d..dc2921db54 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java@@ -335,8 +335,10 @@ public class EcsContainerManager {if (firelensRouter != null) {firelensConfig = firelensConfig(task, taskDef, firelensRouter, firelensLogOptions);firelensExternalConfig = s3ExternalConfig(firelensRouter);- firelensVolumeName = ContainerStorageHelper.dockerName(config, "ecs-firelens-" + taskId);- lifecycleManager.ensureVolume(firelensVolumeName);+ firelensVolumeName = firelensVolumeName(config, task.getTaskArn());+ Map<String, String> labels = new LinkedHashMap<>(ownerLabels());+ labels.put(ContainerStorageHelper.SERVICE_LABEL, "ecs");+ lifecycleManager.ensureVolume(firelensVolumeName, labels);firelensSocketAddress = unixSocketAddress(firelensVolumeName);}} catch (RuntimeException e) {@@ -346,6 +348,14 @@ public class EcsContainerManager {// so without this they would stay valid with nothing behind them until they expire.revokeTaskRoleCredentials(task.getTaskArn());releaseTaskLinkLocalAddresses(task.getTaskArn());+ if (firelensVolumeName != null) {+ EcsTaskHandle partial = new EcsTaskHandle(task.getTaskArn(), Map.of(), Map.of(),+ firelensVolumeName, null, region);+ removeTaskVolumes(partial);+ if (partial.hasTaskVolumes()) {+ throw new EcsTaskLaunchException(e, partial);+ }+ }throw e;}@@ -1438,7 +1448,6 @@ public class EcsContainerManager {cleanupProtectedNetwork(handle);new ArrayList<>(handle.getLogStreamsByContainerId().keySet()).forEach(dockerId -> finalizeLogStream(handle, dockerId));- removeFirelensVolume(handle);revokeTaskRoleCredentials(handle.getTaskArn());if (allRemoved) {removeTaskVolumes(handle);@@ -1529,7 +1538,6 @@ public class EcsContainerManager {terminatedContainerIds.forEach(dockerId -> finalizeLogStream(handle, dockerId));if (handle.allContainersRemoved()) {cleanupProtectedNetwork(handle);- removeFirelensVolume(handle);removeTaskVolumes(handle);// Only now: a container that survived a failed removal still holds its address.releaseTaskLinkLocalAddresses(handle.getTaskArn());@@ -1574,13 +1582,6 @@ public class EcsContainerManager {}}- private void removeFirelensVolume(EcsTaskHandle handle) {- if (handle == null || handle.getFirelensVolumeName() == null) {- return;- }- lifecycleManager.removeVolume(handle.getFirelensVolumeName());- }-private void removeTaskVolumes(EcsTaskHandle handle) {for (String volumeName : handle.getTaskVolumeNames()) {if (lifecycleManager.removeVolume(volumeName)) {@@ -1729,6 +1730,13 @@ public class EcsContainerManager {: FirelensConfigGenerator.fluentBitConfig(ctx);}+ static String firelensVolumeName(EmulatorConfig config, String taskArn) {+ // Keep the host Unix socket path bounded without shortening the ownership labels.+ String identity = ContainerStorageHelper.ownerIdentity(config) + "\n" + taskArn;+ return ContainerStorageHelper.dockerName(null,+ "ecs-firelens-" + Digests.sha256Hex(identity).substring(0, 32));+ }+private String unixSocketAddress(String volumeName) {try {String mountpoint = lifecycleManager.getDockerClient()@@ -1740,6 +1748,10 @@ public class EcsContainerManager {String path = mountpoint.endsWith("/")? mountpoint + "fluent.sock": mountpoint + "/fluent.sock";+ if (path.getBytes(StandardCharsets.UTF_8).length > 107) {+ throw new AwsException("ClientException",+ "FireLens socket path exceeds the Linux Unix socket limit of 107 bytes: " + path, 500);+ }return "unix://" + path;} catch (AwsException e) {throw e;diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.javaindex 19b0f47a26..ac04949acb 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java@@ -74,6 +74,9 @@ public class EcsTaskHandle {this.networkInterfaceId = networkInterfaceId;this.region = region;this.taskVolumeNames = new LinkedHashSet<>(taskVolumeNames);+ if (firelensVolumeName != null) {+ this.taskVolumeNames.add(firelensVolumeName);+ }this.stopTimeouts = new LinkedHashMap<>(stopTimeouts);}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensDockerIntegrationTest.javaindex e403b24be6..e3cfed43a5 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensDockerIntegrationTest.java@@ -5,6 +5,8 @@ import com.github.dockerjava.api.model.Frame;import com.github.dockerjava.api.model.StreamType;import com.github.dockerjava.core.command.ExecStartResultCallback;import com.github.dockerjava.core.command.LogContainerResultCallback;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;import io.github.hectorvent.floci.services.ecs.model.EcsTask;import io.github.hectorvent.floci.services.ecs.model.FirelensConfiguration;@@ -45,6 +47,9 @@ class EcsContainerManagerFirelensDockerIntegrationTest {@InjectDockerClient dockerClient;+ @Inject+ EmulatorConfig config;+private EcsTaskHandle taskHandle;@BeforeEach@@ -92,7 +97,16 @@ class EcsContainerManagerFirelensDockerIntegrationTest {// The Docker daemon, not the application container, makes this connection, so the// address has to be a path on the host: the socket the router creates inside the// task's own volume, which the daemon reaches at the volume's mountpoint.- assertEquals("unix://" + firelensVolumeMountpoint() + "/fluent.sock", appLogDriverAddress(appId));+ String socketPath = firelensVolumeMountpoint() + "/fluent.sock";+ assertTrue(socketPath.getBytes(StandardCharsets.UTF_8).length <= 107);+ assertTrue(taskHandle.getFirelensVolumeName().length() <= 55);+ assertEquals("unix://" + socketPath, appLogDriverAddress(appId));+ Map<String, String> labels = dockerClient.inspectVolumeCmd(taskHandle.getFirelensVolumeName())+ .exec().getLabels();+ assertEquals(ContainerStorageHelper.ownerIdentity(config), labels.get(ContainerStorageHelper.OWNER_LABEL));+ assertEquals("ecs", labels.get(ContainerStorageHelper.SERVICE_LABEL));+ config.docker().resourceNamespace().ifPresent(namespace ->+ assertEquals(namespace, labels.get("floci_namespace")));String routerLogs = awaitRouterLogs(routerId, marker);assertTrue(routerLogs.contains(marker),diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.javaindex 633c91e506..ac4d0751c4 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java@@ -50,9 +50,12 @@ import java.nio.charset.StandardCharsets;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;+import java.util.Optional;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertInstanceOf;+import static org.junit.jupiter.api.Assertions.assertNotEquals;import static org.junit.jupiter.api.Assertions.assertNotSame;import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertSame;@@ -83,6 +86,7 @@ class EcsContainerManagerFirelensTest {private InputStream uploadStream;private byte[] uploadedArchive;private EcsContainerManager manager;+ private EmulatorConfig config;@BeforeEachvoid setUp() {@@ -95,6 +99,7 @@ class EcsContainerManagerFirelensTest {lifecycleManager = mock(ContainerLifecycleManager.class);when(lifecycleManager.resolveImageForLaunch(any(), any())).thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));+ when(lifecycleManager.removeVolume(anyString())).thenReturn(true);when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("app-id", Map.of()));when(lifecycleManager.create(any())).thenReturn("router-id");when(lifecycleManager.startCreated(anyString(), any())).thenReturn(new ContainerInfo("router-id", Map.of()));@@ -120,7 +125,7 @@ class EcsContainerManagerFirelensTest {logStreamer = mock(ContainerLogStreamer.class);when(logStreamer.generateLogStreamName(anyString())).thenReturn("stream");ContainerDetector containerDetector = mock(ContainerDetector.class);- EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);RegionResolver regionResolver = mock(RegionResolver.class);LaunchedContainerAwsEnv awsEnv = mock(LaunchedContainerAwsEnv.class);when(awsEnv.sdkBaselineEnv(any(), any())).thenReturn(List.of());@@ -514,6 +519,88 @@ class EcsContainerManagerFirelensTest {verifyFirelensVolumeRemoved();}+ @Test+ void socketIdentityIncludesTheCompleteOwnerAndTask() {+ String longNamespace = "firelens-" + "namespace".repeat(20);+ when(config.docker().resourceNamespace()).thenReturn(Optional.of(longNamespace));+ String name = EcsContainerManager.firelensVolumeName(config, task().getTaskArn());+ assertEquals(name, EcsContainerManager.firelensVolumeName(config, task().getTaskArn()));+ assertTrue(name.length() <= 55);+ assertNotEquals(name, EcsContainerManager.firelensVolumeName(config, task().getTaskArn() + "other"));+ when(config.docker().resourceNamespace()).thenReturn(Optional.of(longNamespace + "other"));+ assertNotEquals(name, EcsContainerManager.firelensVolumeName(config, task().getTaskArn()));+ when(config.docker().resourceNamespace()).thenReturn(Optional.of(longNamespace));+ when(config.port()).thenReturn(4567);+ assertNotEquals(name, EcsContainerManager.firelensVolumeName(config, task().getTaskArn()));+ }++ @ParameterizedTest+ @ValueSource(strings = {"", " ", "long", "multibyte"})+ void invalidSocketMountpointIsCompensatedBeforeCreatingContainers(String mountpoint) {+ String path = switch (mountpoint) {+ case "long" -> "/" + "x".repeat(108);+ case "multibyte" -> "/" + "é".repeat(54);+ default -> mountpoint;+ };+ when(dockerClient.inspectVolumeCmd(anyString()).exec().getMountpoint()).thenReturn(path);+ assertThrows(AwsException.class, () -> manager.startTask(+ task(), retryTaskDefinition("fluentbit", false, ""), null, "us-east-1"));+ verify(lifecycleManager, never()).create(any());+ verify(lifecycleManager, never()).createAndStart(any());+ verifyFirelensVolumeRemoved();+ }++ @Test+ void failedPreparationCleanupRetainsTheVolumeForRetry() {+ when(dockerClient.inspectVolumeCmd(anyString()).exec().getMountpoint()).thenReturn(null);+ when(lifecycleManager.removeVolume(anyString())).thenReturn(false, true);+ EcsTaskLaunchException failure = assertThrows(EcsTaskLaunchException.class, () -> manager.startTask(+ task(), retryTaskDefinition("fluentbit", false, ""), null, "us-east-1"));+ assertTrue(failure.getHandle().hasTaskVolumes());+ manager.cleanupStoppedTask(failure.getHandle());+ assertFalse(failure.getHandle().hasTaskVolumes());+ }++ @Test+ void socketVolumeKeepsFullOwnershipLabels() {+ when(config.docker().resourceNamespace()).thenReturn(Optional.of("long-owner-namespace"));+ when(config.port()).thenReturn(4566);+ manager.startTask(task(), retryTaskDefinition("fluentbit", false, ""), null, "us-east-1");+ ArgumentCaptor<Map<String, String>> labels = ArgumentCaptor.captor();+ verify(lifecycleManager).ensureVolume(anyString(), labels.capture());+ assertEquals("long-owner-namespace/4566", labels.getValue().get("io.floci.owner"));+ assertEquals("ecs", labels.getValue().get("io.floci.service"));+ assertTrue(labels.getValue().containsKey("io.floci.ecs.run"));+ }++ @ParameterizedTest+ @ValueSource(strings = {"floci-aws-long-namespace-ecs-firelens-abc123",+ "floci-aws-ecs-firelens-043ba6e4a0fd36cf98016b7b4e985125"})+ void handleCleanupRetriesBothLegacyAndBoundedVolumeNames(String volumeName) {+ EcsTaskHandle handle = new EcsTaskHandle(task().getTaskArn(), Map.of(), Map.of(),+ volumeName, null, "us-east-1");+ when(lifecycleManager.removeVolume(volumeName)).thenReturn(false, true);+ manager.cleanupStoppedTask(handle);+ assertTrue(handle.hasTaskVolumes());+ manager.cleanupStoppedTask(handle);+ assertFalse(handle.hasTaskVolumes());+ }++ @ParameterizedTest+ @CsvSource({"107,true", "108,false"})+ void socketLengthChecksTheCompleteHostPath(int bytes, boolean accepted) {+ String mountpoint = "/" + "x".repeat(bytes - 13);+ when(dockerClient.inspectVolumeCmd(anyString()).exec().getMountpoint()).thenReturn(mountpoint);+ if (accepted) {+ manager.startTask(task(), retryTaskDefinition("fluentbit", false, ""), null, "us-east-1");+ } else {+ assertThrows(AwsException.class, () -> manager.startTask(+ task(), retryTaskDefinition("fluentbit", false, ""), null, "us-east-1"));+ verify(lifecycleManager, never()).create(any());+ verifyFirelensVolumeRemoved();+ }+ }+private DockerArchiveUploadFixture failingUploads() {DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(dockerClient, "router-id");uploads.failures = Integer.MAX_VALUE;@@ -532,7 +619,7 @@ class EcsContainerManagerFirelensTest {private void verifyFirelensVolumeRemoved() {ArgumentCaptor<String> volumeName = ArgumentCaptor.forClass(String.class);- verify(lifecycleManager).ensureVolume(volumeName.capture());+ verify(lifecycleManager).ensureVolume(volumeName.capture(), any());verify(lifecycleManager).removeVolume(volumeName.getValue());}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.javaindex 1f2e177555..6fd2b58005 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java@@ -37,6 +37,7 @@ import java.util.Map;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyInt;import static org.mockito.ArgumentMatchers.anyString;@@ -134,7 +135,7 @@ class EcsContainerManagerVolumesFromTest {when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);InspectVolumeCmd inspectVolumeCmd = mock(InspectVolumeCmd.class);InspectVolumeResponse volume = mock(InspectVolumeResponse.class);- when(dockerClient.inspectVolumeCmd("floci-aws-ecs-firelens-volumesfrom1")).thenReturn(inspectVolumeCmd);+ when(dockerClient.inspectVolumeCmd(anyString())).thenReturn(inspectVolumeCmd);when(inspectVolumeCmd.exec()).thenReturn(volume);when(volume.getMountpoint()).thenReturn("/var/lib/docker/volumes/floci-ecs-firelens-volumesfrom1/_data");CopyArchiveToContainerCmd copyCmd = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);@@ -166,8 +167,8 @@ class EcsContainerManagerVolumesFromTest {lifecycleOrder.verify(lifecycleManager).createAndStart(any(ContainerSpec.class));verify(routerBuilder).withVolumesFrom("source-id", false);verify(appBuilder).withVolumesFrom("source-id", true);- verify(lifecycleManager).ensureVolume("floci-aws-ecs-firelens-volumesfrom1");- verify(routerBuilder).withNamedVolume("floci-aws-ecs-firelens-volumesfrom1", "/var/run");+ verify(lifecycleManager).ensureVolume(eq(handle.getFirelensVolumeName()), any());+ verify(routerBuilder).withNamedVolume(handle.getFirelensVolumeName(), "/var/run");verify(copyCmd).withRemotePath("/fluent-bit/etc");verify(routerBuilder, never()).withLoopbackPortBinding(anyInt(), anyInt());verify(appBuilder, never()).withNetworkMode(anyString());@@ -184,7 +185,7 @@ class EcsContainerManagerVolumesFromTest {verify(logStreamer, never()).attachForAccount(any(), eq("app-id"), any(), any(), any(), any());verify(logStreamer).attachForAccount(eq("000000000000"), eq("source-id"), any(), any(), eq("us-east-1"), any());verify(logStreamer).attachForAccount(eq("000000000000"), eq("router-id"), any(), any(), eq("us-east-1"), any());- assertEquals("floci-aws-ecs-firelens-volumesfrom1", handle.getFirelensVolumeName());+ assertTrue(handle.getFirelensVolumeName().length() <= 55);assertEquals(List.of("source", "router", "app"), handle.getContainerIds().keySet().stream().toList());assertEquals(List.of("app", "router", "source"),ecsTask.getContainers().stream().map(Container::getName).toList());@@ -206,7 +207,7 @@ class EcsContainerManagerVolumesFromTest {verify(containerBuilder, never()).newContainer(anyString());verify(lifecycleManager, never()).create(any());verify(lifecycleManager, never()).createAndStart(any());- verify(lifecycleManager, never()).ensureVolume(anyString());+ verify(lifecycleManager, never()).ensureVolume(anyString(), any());}@Test@@ -235,7 +236,7 @@ class EcsContainerManagerVolumesFromTest {verify(containerBuilder, never()).newContainer(anyString());verify(lifecycleManager, never()).create(any());verify(lifecycleManager, never()).createAndStart(any());- verify(lifecycleManager, never()).ensureVolume(anyString());+ verify(lifecycleManager, never()).ensureVolume(anyString(), any());}@Test