Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ecs): bound FireLens socket volume paths (#6787)

Andre Kurait2d3fb0ce8764
diff --git a/docs/services/ecs.md b/docs/services/ecs.md
index c222118a11..c0df41e151 100644
--- a/docs/services/ecs.md
+++ b/docs/services/ecs.md
@@ -196,6 +196,12 @@ other case, the injected `FLUENT_HOST` (the router's container IP) must be reach
config is written to `/fluent-bit/etc/fluent-bit.conf`. Fluentd config is
written to `/fluentd/etc/fluent.conf` and uses `@type` (not `Name`) for output plugins; Floci
does not inject an `endpoint` into Fluentd outputs.
+FireLens uses a bounded socket-volume name derived from the complete deployment and task identity,
+while its Docker labels retain the full configured resource namespace. The inspected host socket
+path must fit Linux's 107-byte Unix socket pathname limit. An unusually long Docker data root can
+still exceed that limit; launch then fails before creating containers and retains any failed volume
+cleanup for retry. Operator-named volumes are unaffected.
+
FireLens configuration archive uploads retry transient Docker connection failures up to six
attempts, with exponential backoff starting at 500 ms. Each attempt sends the complete prebuilt
archive through a fresh byte-backed stream, without a streaming pipe or writer thread.
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
index 4bc5d1111d..dc2921db54 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
@@ -335,8 +335,10 @@ public class EcsContainerManager {
if (firelensRouter != null) {
firelensConfig = firelensConfig(task, taskDef, firelensRouter, firelensLogOptions);
firelensExternalConfig = s3ExternalConfig(firelensRouter);
- firelensVolumeName = ContainerStorageHelper.dockerName(config, "ecs-firelens-" + taskId);
- lifecycleManager.ensureVolume(firelensVolumeName);
+ firelensVolumeName = firelensVolumeName(config, task.getTaskArn());
+ Map<String, String> labels = new LinkedHashMap<>(ownerLabels());
+ labels.put(ContainerStorageHelper.SERVICE_LABEL, "ecs");
+ lifecycleManager.ensureVolume(firelensVolumeName, labels);
firelensSocketAddress = unixSocketAddress(firelensVolumeName);
}
} catch (RuntimeException e) {
@@ -346,6 +348,14 @@ public class EcsContainerManager {
// so without this they would stay valid with nothing behind them until they expire.
revokeTaskRoleCredentials(task.getTaskArn());
releaseTaskLinkLocalAddresses(task.getTaskArn());
+ if (firelensVolumeName != null) {
+ EcsTaskHandle partial = new EcsTaskHandle(task.getTaskArn(), Map.of(), Map.of(),
+ firelensVolumeName, null, region);
+ removeTaskVolumes(partial);
+ if (partial.hasTaskVolumes()) {
+ throw new EcsTaskLaunchException(e, partial);
+ }
+ }
throw e;
}
@@ -1438,7 +1448,6 @@ public class EcsContainerManager {
cleanupProtectedNetwork(handle);
new ArrayList<>(handle.getLogStreamsByContainerId().keySet())
.forEach(dockerId -> finalizeLogStream(handle, dockerId));
- removeFirelensVolume(handle);
revokeTaskRoleCredentials(handle.getTaskArn());
if (allRemoved) {
removeTaskVolumes(handle);
@@ -1529,7 +1538,6 @@ public class EcsContainerManager {
terminatedContainerIds.forEach(dockerId -> finalizeLogStream(handle, dockerId));
if (handle.allContainersRemoved()) {
cleanupProtectedNetwork(handle);
- removeFirelensVolume(handle);
removeTaskVolumes(handle);
// Only now: a container that survived a failed removal still holds its address.
releaseTaskLinkLocalAddresses(handle.getTaskArn());
@@ -1574,13 +1582,6 @@ public class EcsContainerManager {
}
}
- private void removeFirelensVolume(EcsTaskHandle handle) {
- if (handle == null || handle.getFirelensVolumeName() == null) {
- return;
- }
- lifecycleManager.removeVolume(handle.getFirelensVolumeName());
- }
-
private void removeTaskVolumes(EcsTaskHandle handle) {
for (String volumeName : handle.getTaskVolumeNames()) {
if (lifecycleManager.removeVolume(volumeName)) {
@@ -1729,6 +1730,13 @@ public class EcsContainerManager {
: FirelensConfigGenerator.fluentBitConfig(ctx);
}
+ static String firelensVolumeName(EmulatorConfig config, String taskArn) {
+ // Keep the host Unix socket path bounded without shortening the ownership labels.
+ String identity = ContainerStorageHelper.ownerIdentity(config) + "\n" + taskArn;
+ return ContainerStorageHelper.dockerName(null,
+ "ecs-firelens-" + Digests.sha256Hex(identity).substring(0, 32));
+ }
+
private String unixSocketAddress(String volumeName) {
try {
String mountpoint = lifecycleManager.getDockerClient()
@@ -1740,6 +1748,10 @@ public class EcsContainerManager {
String path = mountpoint.endsWith("/")
? mountpoint + "fluent.sock"
: mountpoint + "/fluent.sock";
+ if (path.getBytes(StandardCharsets.UTF_8).length > 107) {
+ throw new AwsException("ClientException",
+ "FireLens socket path exceeds the Linux Unix socket limit of 107 bytes: " + path, 500);
+ }
return "unix://" + path;
} catch (AwsException e) {
throw e;
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java
index 19b0f47a26..ac04949acb 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java
@@ -74,6 +74,9 @@ public class EcsTaskHandle {
this.networkInterfaceId = networkInterfaceId;
this.region = region;
this.taskVolumeNames = new LinkedHashSet<>(taskVolumeNames);
+ if (firelensVolumeName != null) {
+ this.taskVolumeNames.add(firelensVolumeName);
+ }
this.stopTimeouts = new LinkedHashMap<>(stopTimeouts);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensDockerIntegrationTest.java
index e403b24be6..e3cfed43a5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensDockerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensDockerIntegrationTest.java
@@ -5,6 +5,8 @@ import com.github.dockerjava.api.model.Frame;
import com.github.dockerjava.api.model.StreamType;
import com.github.dockerjava.core.command.ExecStartResultCallback;
import com.github.dockerjava.core.command.LogContainerResultCallback;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
import io.github.hectorvent.floci.services.ecs.model.EcsTask;
import io.github.hectorvent.floci.services.ecs.model.FirelensConfiguration;
@@ -45,6 +47,9 @@ class EcsContainerManagerFirelensDockerIntegrationTest {
@Inject
DockerClient dockerClient;
+ @Inject
+ EmulatorConfig config;
+
private EcsTaskHandle taskHandle;
@BeforeEach
@@ -92,7 +97,16 @@ class EcsContainerManagerFirelensDockerIntegrationTest {
// The Docker daemon, not the application container, makes this connection, so the
// address has to be a path on the host: the socket the router creates inside the
// task's own volume, which the daemon reaches at the volume's mountpoint.
- assertEquals("unix://" + firelensVolumeMountpoint() + "/fluent.sock", appLogDriverAddress(appId));
+ String socketPath = firelensVolumeMountpoint() + "/fluent.sock";
+ assertTrue(socketPath.getBytes(StandardCharsets.UTF_8).length <= 107);
+ assertTrue(taskHandle.getFirelensVolumeName().length() <= 55);
+ assertEquals("unix://" + socketPath, appLogDriverAddress(appId));
+ Map<String, String> labels = dockerClient.inspectVolumeCmd(taskHandle.getFirelensVolumeName())
+ .exec().getLabels();
+ assertEquals(ContainerStorageHelper.ownerIdentity(config), labels.get(ContainerStorageHelper.OWNER_LABEL));
+ assertEquals("ecs", labels.get(ContainerStorageHelper.SERVICE_LABEL));
+ config.docker().resourceNamespace().ifPresent(namespace ->
+ assertEquals(namespace, labels.get("floci_namespace")));
String routerLogs = awaitRouterLogs(routerId, marker);
assertTrue(routerLogs.contains(marker),
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java
index 633c91e506..ac4d0751c4 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java
@@ -50,9 +50,12 @@ import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
@@ -83,6 +86,7 @@ class EcsContainerManagerFirelensTest {
private InputStream uploadStream;
private byte[] uploadedArchive;
private EcsContainerManager manager;
+ private EmulatorConfig config;
@BeforeEach
void setUp() {
@@ -95,6 +99,7 @@ class EcsContainerManagerFirelensTest {
lifecycleManager = mock(ContainerLifecycleManager.class);
when(lifecycleManager.resolveImageForLaunch(any(), any()))
.thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));
+ when(lifecycleManager.removeVolume(anyString())).thenReturn(true);
when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerInfo("app-id", Map.of()));
when(lifecycleManager.create(any())).thenReturn("router-id");
when(lifecycleManager.startCreated(anyString(), any())).thenReturn(new ContainerInfo("router-id", Map.of()));
@@ -120,7 +125,7 @@ class EcsContainerManagerFirelensTest {
logStreamer = mock(ContainerLogStreamer.class);
when(logStreamer.generateLogStreamName(anyString())).thenReturn("stream");
ContainerDetector containerDetector = mock(ContainerDetector.class);
- EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
RegionResolver regionResolver = mock(RegionResolver.class);
LaunchedContainerAwsEnv awsEnv = mock(LaunchedContainerAwsEnv.class);
when(awsEnv.sdkBaselineEnv(any(), any())).thenReturn(List.of());
@@ -514,6 +519,88 @@ class EcsContainerManagerFirelensTest {
verifyFirelensVolumeRemoved();
}
+ @Test
+ void socketIdentityIncludesTheCompleteOwnerAndTask() {
+ String longNamespace = "firelens-" + "namespace".repeat(20);
+ when(config.docker().resourceNamespace()).thenReturn(Optional.of(longNamespace));
+ String name = EcsContainerManager.firelensVolumeName(config, task().getTaskArn());
+ assertEquals(name, EcsContainerManager.firelensVolumeName(config, task().getTaskArn()));
+ assertTrue(name.length() <= 55);
+ assertNotEquals(name, EcsContainerManager.firelensVolumeName(config, task().getTaskArn() + "other"));
+ when(config.docker().resourceNamespace()).thenReturn(Optional.of(longNamespace + "other"));
+ assertNotEquals(name, EcsContainerManager.firelensVolumeName(config, task().getTaskArn()));
+ when(config.docker().resourceNamespace()).thenReturn(Optional.of(longNamespace));
+ when(config.port()).thenReturn(4567);
+ assertNotEquals(name, EcsContainerManager.firelensVolumeName(config, task().getTaskArn()));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"", " ", "long", "multibyte"})
+ void invalidSocketMountpointIsCompensatedBeforeCreatingContainers(String mountpoint) {
+ String path = switch (mountpoint) {
+ case "long" -> "/" + "x".repeat(108);
+ case "multibyte" -> "/" + "é".repeat(54);
+ default -> mountpoint;
+ };
+ when(dockerClient.inspectVolumeCmd(anyString()).exec().getMountpoint()).thenReturn(path);
+ assertThrows(AwsException.class, () -> manager.startTask(
+ task(), retryTaskDefinition("fluentbit", false, ""), null, "us-east-1"));
+ verify(lifecycleManager, never()).create(any());
+ verify(lifecycleManager, never()).createAndStart(any());
+ verifyFirelensVolumeRemoved();
+ }
+
+ @Test
+ void failedPreparationCleanupRetainsTheVolumeForRetry() {
+ when(dockerClient.inspectVolumeCmd(anyString()).exec().getMountpoint()).thenReturn(null);
+ when(lifecycleManager.removeVolume(anyString())).thenReturn(false, true);
+ EcsTaskLaunchException failure = assertThrows(EcsTaskLaunchException.class, () -> manager.startTask(
+ task(), retryTaskDefinition("fluentbit", false, ""), null, "us-east-1"));
+ assertTrue(failure.getHandle().hasTaskVolumes());
+ manager.cleanupStoppedTask(failure.getHandle());
+ assertFalse(failure.getHandle().hasTaskVolumes());
+ }
+
+ @Test
+ void socketVolumeKeepsFullOwnershipLabels() {
+ when(config.docker().resourceNamespace()).thenReturn(Optional.of("long-owner-namespace"));
+ when(config.port()).thenReturn(4566);
+ manager.startTask(task(), retryTaskDefinition("fluentbit", false, ""), null, "us-east-1");
+ ArgumentCaptor<Map<String, String>> labels = ArgumentCaptor.captor();
+ verify(lifecycleManager).ensureVolume(anyString(), labels.capture());
+ assertEquals("long-owner-namespace/4566", labels.getValue().get("io.floci.owner"));
+ assertEquals("ecs", labels.getValue().get("io.floci.service"));
+ assertTrue(labels.getValue().containsKey("io.floci.ecs.run"));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"floci-aws-long-namespace-ecs-firelens-abc123",
+ "floci-aws-ecs-firelens-043ba6e4a0fd36cf98016b7b4e985125"})
+ void handleCleanupRetriesBothLegacyAndBoundedVolumeNames(String volumeName) {
+ EcsTaskHandle handle = new EcsTaskHandle(task().getTaskArn(), Map.of(), Map.of(),
+ volumeName, null, "us-east-1");
+ when(lifecycleManager.removeVolume(volumeName)).thenReturn(false, true);
+ manager.cleanupStoppedTask(handle);
+ assertTrue(handle.hasTaskVolumes());
+ manager.cleanupStoppedTask(handle);
+ assertFalse(handle.hasTaskVolumes());
+ }
+
+ @ParameterizedTest
+ @CsvSource({"107,true", "108,false"})
+ void socketLengthChecksTheCompleteHostPath(int bytes, boolean accepted) {
+ String mountpoint = "/" + "x".repeat(bytes - 13);
+ when(dockerClient.inspectVolumeCmd(anyString()).exec().getMountpoint()).thenReturn(mountpoint);
+ if (accepted) {
+ manager.startTask(task(), retryTaskDefinition("fluentbit", false, ""), null, "us-east-1");
+ } else {
+ assertThrows(AwsException.class, () -> manager.startTask(
+ task(), retryTaskDefinition("fluentbit", false, ""), null, "us-east-1"));
+ verify(lifecycleManager, never()).create(any());
+ verifyFirelensVolumeRemoved();
+ }
+ }
+
private DockerArchiveUploadFixture failingUploads() {
DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(dockerClient, "router-id");
uploads.failures = Integer.MAX_VALUE;
@@ -532,7 +619,7 @@ class EcsContainerManagerFirelensTest {
private void verifyFirelensVolumeRemoved() {
ArgumentCaptor<String> volumeName = ArgumentCaptor.forClass(String.class);
- verify(lifecycleManager).ensureVolume(volumeName.capture());
+ verify(lifecycleManager).ensureVolume(volumeName.capture(), any());
verify(lifecycleManager).removeVolume(volumeName.getValue());
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java
index 1f2e177555..6fd2b58005 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java
@@ -37,6 +37,7 @@ import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
@@ -134,7 +135,7 @@ class EcsContainerManagerVolumesFromTest {
when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
InspectVolumeCmd inspectVolumeCmd = mock(InspectVolumeCmd.class);
InspectVolumeResponse volume = mock(InspectVolumeResponse.class);
- when(dockerClient.inspectVolumeCmd("floci-aws-ecs-firelens-volumesfrom1")).thenReturn(inspectVolumeCmd);
+ when(dockerClient.inspectVolumeCmd(anyString())).thenReturn(inspectVolumeCmd);
when(inspectVolumeCmd.exec()).thenReturn(volume);
when(volume.getMountpoint()).thenReturn("/var/lib/docker/volumes/floci-ecs-firelens-volumesfrom1/_data");
CopyArchiveToContainerCmd copyCmd = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);
@@ -166,8 +167,8 @@ class EcsContainerManagerVolumesFromTest {
lifecycleOrder.verify(lifecycleManager).createAndStart(any(ContainerSpec.class));
verify(routerBuilder).withVolumesFrom("source-id", false);
verify(appBuilder).withVolumesFrom("source-id", true);
- verify(lifecycleManager).ensureVolume("floci-aws-ecs-firelens-volumesfrom1");
- verify(routerBuilder).withNamedVolume("floci-aws-ecs-firelens-volumesfrom1", "/var/run");
+ verify(lifecycleManager).ensureVolume(eq(handle.getFirelensVolumeName()), any());
+ verify(routerBuilder).withNamedVolume(handle.getFirelensVolumeName(), "/var/run");
verify(copyCmd).withRemotePath("/fluent-bit/etc");
verify(routerBuilder, never()).withLoopbackPortBinding(anyInt(), anyInt());
verify(appBuilder, never()).withNetworkMode(anyString());
@@ -184,7 +185,7 @@ class EcsContainerManagerVolumesFromTest {
verify(logStreamer, never()).attachForAccount(any(), eq("app-id"), any(), any(), any(), any());
verify(logStreamer).attachForAccount(eq("000000000000"), eq("source-id"), any(), any(), eq("us-east-1"), any());
verify(logStreamer).attachForAccount(eq("000000000000"), eq("router-id"), any(), any(), eq("us-east-1"), any());
- assertEquals("floci-aws-ecs-firelens-volumesfrom1", handle.getFirelensVolumeName());
+ assertTrue(handle.getFirelensVolumeName().length() <= 55);
assertEquals(List.of("source", "router", "app"), handle.getContainerIds().keySet().stream().toList());
assertEquals(List.of("app", "router", "source"),
ecsTask.getContainers().stream().map(Container::getName).toList());
@@ -206,7 +207,7 @@ class EcsContainerManagerVolumesFromTest {
verify(containerBuilder, never()).newContainer(anyString());
verify(lifecycleManager, never()).create(any());
verify(lifecycleManager, never()).createAndStart(any());
- verify(lifecycleManager, never()).ensureVolume(anyString());
+ verify(lifecycleManager, never()).ensureVolume(anyString(), any());
}
@Test
@@ -235,7 +236,7 @@ class EcsContainerManagerVolumesFromTest {
verify(containerBuilder, never()).newContainer(anyString());
verify(lifecycleManager, never()).create(any());
verify(lifecycleManager, never()).createAndStart(any());
- verify(lifecycleManager, never()).ensureVolume(anyString());
+ verify(lifecycleManager, never()).ensureVolume(anyString(), any());
}
@Test