Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ecs): send container logs to the awslogs group and stream from the task definition (#4808)
Aymen JAMAAOUIbc2592e300e4
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsRunningTaskTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsRunningTaskTest.javaindex 145c7cd1fa..b11c20ec99 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsRunningTaskTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/EcsRunningTaskTest.java@@ -15,6 +15,8 @@ import software.amazon.awssdk.services.ecs.model.CreateClusterRequest;import software.amazon.awssdk.services.ecs.model.DeleteClusterRequest;import software.amazon.awssdk.services.ecs.model.ExecuteCommandRequest;import software.amazon.awssdk.services.ecs.model.LaunchType;+import software.amazon.awssdk.services.ecs.model.LogConfiguration;+import software.amazon.awssdk.services.ecs.model.LogDriver;import software.amazon.awssdk.services.ecs.model.NetworkMode;import software.amazon.awssdk.services.ecs.model.RegisterTaskDefinitionRequest;import software.amazon.awssdk.services.ecs.model.RunTaskRequest;@@ -29,6 +31,7 @@ import java.nio.charset.StandardCharsets;import java.security.MessageDigest;import java.util.Arrays;import java.util.List;+import java.util.Map;import java.util.UUID;import java.util.concurrent.BlockingQueue;import java.util.concurrent.CompletionStage;@@ -69,6 +72,13 @@ class EcsRunningTaskTest {+ "$(wget -qO- $ECS_CONTAINER_METADATA_URI_V4/stats); sleep 120").essential(true).memory(64)+ .logConfiguration(LogConfiguration.builder()+ .logDriver(LogDriver.AWSLOGS)+ .options(Map.of(+ "awslogs-group", "/ecs/" + family,+ "awslogs-region", logs.serviceClientConfiguration().region().id(),+ "awslogs-stream-prefix", "ecs"))+ .build()).build()).build());taskArn = ecs.runTask(RunTaskRequest.builder()diff --git a/docs/services/ecs.md b/docs/services/ecs.mdindex e7149a3321..b6a08ec0cb 100644--- a/docs/services/ecs.md+++ b/docs/services/ecs.md@@ -131,8 +131,13 @@ port `24224`, is rejected at launch. A `fluentbit` or `fluentd` FireLenscontainer is acted on at launch: Floci generates the router config (unix socket input, TCP forwardon bridge/awsvpc, ECS metadata, optional include of a `config-file-type=file` or `s3` extraconfig, and one output per `awsfirelens` container), starts that router first, and points application-containers with `logDriver: awsfirelens` at the generated unix socket. Other log drivers,-including `awslogs`, still stream to CloudWatch via Floci rather than the configured driver.+containers with `logDriver: awsfirelens` at the generated unix socket. A container with+`logDriver: awslogs` streams to CloudWatch Logs through Floci, in the `awslogs-group` log group+and the `awslogs-region` region (the task's region when it is not set). The log stream is+`<awslogs-stream-prefix>/<container-name>/<task-id>`, or the Docker container ID when no prefix is+set, as on AWS. Floci creates the log group when it does not exist, even without+`awslogs-create-group`. A container with no `logConfiguration`, or with another log driver, sends+nothing to CloudWatch Logs; its output still appears in Floci's own log.An `[OUTPUT]` for an AWS destination whose plugin reads a URL from `endpoint` (`s3`,`cloudwatch`, `firehose`) also gets `Endpoint` set to Floci's container-reachable base URL. TheFluent Bit AWS plugins take a custom endpoint only from their own configuration and ignore thediff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.javaindex c0df117dc3..b57cd4f64c 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java@@ -112,7 +112,25 @@ public class ContainerLogStreamer {forwardToCloudWatchLogs(accountId, logGroup, logStream, region, trimmed);}};+ return follow(containerId, since, emitter);+ }++ /**+ * Like {@link #attach}, but sends the output only to the console logger, not to CloudWatch Logs.+ * Use it for a container whose output must stay visible locally but has no CloudWatch+ * destination, such as an ECS container without an {@code awslogs} log configuration.+ */+ public Closeable attachConsoleOnly(String containerId, String logPrefix) {+ Consumer<String> emitter = line -> {+ String trimmed = line.stripTrailing();+ if (!trimmed.isEmpty()) {+ LOG.infov("[{0}] {1}", logPrefix, trimmed);+ }+ };+ return follow(containerId, null, emitter);+ }+ private Closeable follow(String containerId, Instant since, Consumer<String> emitter) {try {LogReassemblyCallback callback = new LogReassemblyCallback(emitter);LogContainerCmd command = dockerClient.logContainerCmd(containerId)diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.javaindex 1344dbde8a..35033b03d7 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java@@ -509,12 +509,14 @@ public class EcsContainerManager {containerIds.put(def.getName(), dockerId);// awsfirelens containers are shipped to Fluent Bit by Docker; don't also scrape json-file.+ // Only awslogs containers go to CloudWatch Logs; the others stay on the console.if (!awsFirelens) {- String logGroup = "/ecs/" + taskDef.getFamily();- String logStream = logStreamer.generateLogStreamName(def.getName() + "/" + taskId);- Closeable logHandle = logStreamer.attach(- dockerId, logGroup, logStream, region,- "ecs:" + taskDef.getFamily() + ":" + def.getName());+ String logPrefix = "ecs:" + taskDef.getFamily() + ":" + def.getName();+ AwsLogsDestination awsLogs = awsLogsDestination(def, taskId, dockerId, region);+ Closeable logHandle = awsLogs == null+ ? logStreamer.attachConsoleOnly(dockerId, logPrefix)+ : logStreamer.attach(+ dockerId, awsLogs.group(), awsLogs.stream(), awsLogs.region(), logPrefix);if (logHandle != null) {logStreamsByContainerId.put(dockerId, logHandle);}@@ -1463,6 +1465,35 @@ public class EcsContainerManager {return log != null && "awsfirelens".equals(log.logDriver());}+ private record AwsLogsDestination(String group, String stream, String region) {}++ /**+ * Where a container's output goes in CloudWatch Logs. Only the {@code awslogs} driver sends it+ * there, to its {@code awslogs-group}. The stream is {@code prefix/container-name/task-id}, or the+ * Docker container ID when {@code awslogs-stream-prefix} is not set, as on AWS. Returns null when+ * nothing must be sent.+ */+ private static AwsLogsDestination awsLogsDestination(+ ContainerDefinition def, String taskId, String dockerId, String taskRegion) {+ LogConfiguration log = def.getLogConfiguration();+ if (log == null || !"awslogs".equals(log.logDriver())) {+ return null;+ }+ Map<String, String> options = log.options() == null ? Map.of() : log.options();+ String group = options.get("awslogs-group");+ if (group == null || group.isBlank()) {+ LOG.warnv("ECS task {0} container {1} uses awslogs without awslogs-group; its logs are not sent to CloudWatch Logs",+ taskId, def.getName());+ return null;+ }+ String prefix = options.get("awslogs-stream-prefix");+ String stream = prefix == null || prefix.isBlank()+ ? dockerId+ : prefix + "/" + def.getName() + "/" + taskId;+ String region = options.get("awslogs-region");+ return new AwsLogsDestination(group, stream, region == null || region.isBlank() ? taskRegion : region);+ }+private static List<ContainerDefinition> launchOrder(List<ContainerDefinition> defs, ContainerDefinition firelensRouter) {if (firelensRouter == null) {diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.javaindex 6c0fc1157c..1951bb17fc 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java@@ -106,6 +106,27 @@ class ContainerLogStreamerTest {eq("us-east-1"));}+ @Test+ void consoleOnlyStreamReachesTheConsoleButNotCloudWatch() {+ DockerClient dockerClient = mock(DockerClient.class);+ LogContainerCmd command = mock(LogContainerCmd.class, RETURNS_SELF);+ when(dockerClient.logContainerCmd("container-1")).thenReturn(command);+ CloudWatchLogsService cloudWatchLogsService = mock(CloudWatchLogsService.class);+ ContainerLogStreamer streamer = new ContainerLogStreamer(dockerClient, cloudWatchLogsService);++ streamer.attachConsoleOnly("container-1", "ecs:family:main");+ ArgumentCaptor<ContainerLogStreamer.LogReassemblyCallback> callback =+ ArgumentCaptor.forClass(ContainerLogStreamer.LogReassemblyCallback.class);+ verify(command).exec(callback.capture());++ List<LogRecord> records = LogCapture.capture(ContainerLogStreamer.class, () ->+ callback.getValue().onNext(new Frame(StreamType.STDOUT, utf8("container output\n"))));++ assertEquals(1, records.size());+ verify(cloudWatchLogsService, never()).createLogGroupForAccount(any(), any(), any(), any(), any());+ verify(cloudWatchLogsService, never()).putLogEventsForAccount(any(), any(), any(), anyList(), any());+ }+@ParameterizedTest@ValueSource(booleans = {true, false})void execStreamAlwaysReachesCloudWatchAndConsoleIsOptional(boolean logToConsole) {diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.javaindex 977f58d2bf..97e879644e 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java@@ -114,6 +114,8 @@ class EcsContainerManagerFirelensTest {router.setName("log_router");router.setImage("amazon/aws-for-fluent-bit:stable");router.setFirelensConfiguration(new FirelensConfiguration("fluentbit", Map.of()));+ router.setLogConfiguration(new LogConfiguration("awslogs",+ Map.of("awslogs-group", "/ecs/firelens-family", "awslogs-stream-prefix", "firelens"), null));ContainerDefinition app = new ContainerDefinition();app.setName("app");@@ -150,6 +152,7 @@ class EcsContainerManagerFirelensTest {verify(logStreamer).attach(eq("router-id"), anyString(), anyString(), anyString(), anyString());verify(logStreamer, never()).attach(eq("app-id"), anyString(), anyString(), anyString(), anyString());+ verify(logStreamer, never()).attachConsoleOnly(eq("app-id"), anyString());assertTrue(handle.getFirelensVolumeName().contains("firelens"));verify(copyCmd).withRemotePath("/fluent-bit/etc");diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLogConfigurationTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLogConfigurationTest.javanew file mode 100644index 0000000000..db2bf7ac50--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLogConfigurationTest.java@@ -0,0 +1,139 @@+package io.github.hectorvent.floci.services.ecs.container;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;+import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;+import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;+import io.github.hectorvent.floci.services.ecs.model.EcsTask;+import io.github.hectorvent.floci.services.ecs.model.LogConfiguration;+import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService.LaunchImage;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;+import io.github.hectorvent.floci.services.ssm.SsmService;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.List;+import java.util.Map;++import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.RETURNS_SELF;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++/**+ * A container's output reaches CloudWatch Logs only through the {@code awslogs} driver, in the+ * group and stream its log configuration names. Other non-FireLens containers stay on the console.+ * Docker is mocked, so no daemon is needed.+ */+class EcsContainerManagerLogConfigurationTest {++ private static final String TASK_ARN = "arn:aws:ecs:us-east-1:000000000000:task/test-cluster/abc123";++ private ContainerLogStreamer logStreamer;+ private EcsContainerManager manager;++ @BeforeEach+ void setUp() {+ ContainerBuilder.Builder builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);+ ContainerBuilder containerBuilder = mock(ContainerBuilder.class);+ when(containerBuilder.newContainer(anyString())).thenReturn(builder);+ when(containerBuilder.resolveImage(anyString())).thenAnswer(invocation -> invocation.getArgument(0));++ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ when(lifecycleManager.resolveImageForLaunch(any(), any()))+ .thenAnswer(invocation -> new LaunchImage(invocation.getArgument(0), null));+ when(lifecycleManager.createAndStart(any()))+ .thenReturn(new ContainerInfo("docker-id", Map.of()));++ logStreamer = mock(ContainerLogStreamer.class);+ LaunchedContainerAwsEnv awsEnv = mock(LaunchedContainerAwsEnv.class);+ when(awsEnv.sdkBaselineEnv(any(), any())).thenReturn(List.of());+ EcrRegistryManager ecrRegistryManager = mock(EcrRegistryManager.class);+ when(ecrRegistryManager.rewriteImageUri(anyString())).thenAnswer(inv -> inv.getArgument(0));++ manager = new EcsContainerManager(containerBuilder, lifecycleManager, logStreamer,+ mock(ContainerDetector.class), mock(EmulatorConfig.class, RETURNS_DEEP_STUBS),+ mock(RegionResolver.class), awsEnv, mock(SsmService.class), mock(SecretsManagerService.class),+ mock(S3Service.class), ecrRegistryManager, mock(HostVolumePolicy.class));+ }++ @Test+ void awslogsSendsToTheConfiguredGroupAndPrefixedStream() {+ startTask(new LogConfiguration("awslogs", Map.of(+ "awslogs-group", "my-app",+ "awslogs-region", "us-east-1",+ "awslogs-stream-prefix", "web"), null));++ verify(logStreamer).attach(eq("docker-id"), eq("my-app"), eq("web/main/abc123"), eq("us-east-1"), anyString());+ }++ @Test+ void awslogsWithoutStreamPrefixNamesTheStreamAfterTheDockerContainer() {+ startTask(new LogConfiguration("awslogs", Map.of("awslogs-group", "my-app"), null));++ verify(logStreamer).attach(eq("docker-id"), eq("my-app"), eq("docker-id"), eq("us-east-1"), anyString());+ }++ @Test+ void awslogsRegionOptionChoosesTheLogsRegion() {+ startTask(new LogConfiguration("awslogs", Map.of(+ "awslogs-group", "my-app",+ "awslogs-region", "eu-west-1",+ "awslogs-stream-prefix", "web"), null));++ verify(logStreamer).attach(eq("docker-id"), eq("my-app"), eq("web/main/abc123"), eq("eu-west-1"), anyString());+ }++ @Test+ void containerWithoutLogConfigurationOnlyStreamsToTheConsole() {+ startTask(null);++ verify(logStreamer).attachConsoleOnly("docker-id", "ecs:logs-family:main");+ verify(logStreamer, never()).attach(any(), any(), any(), any(), any());+ }++ @Test+ void otherLogDriverOnlyStreamsToTheConsole() {+ startTask(new LogConfiguration("splunk", Map.of("splunk-url", "https://splunk.example.com"), null));++ verify(logStreamer).attachConsoleOnly("docker-id", "ecs:logs-family:main");+ verify(logStreamer, never()).attach(any(), any(), any(), any(), any());+ }++ @Test+ void awslogsWithoutGroupOnlyStreamsToTheConsole() {+ startTask(new LogConfiguration("awslogs", Map.of("awslogs-stream-prefix", "web"), null));++ verify(logStreamer).attachConsoleOnly("docker-id", "ecs:logs-family:main");+ verify(logStreamer, never()).attach(any(), any(), any(), any(), any());+ }++ private void startTask(LogConfiguration logConfiguration) {+ ContainerDefinition main = new ContainerDefinition();+ main.setName("main");+ main.setImage("busybox:latest");+ main.setLogConfiguration(logConfiguration);++ TaskDefinition taskDef = new TaskDefinition();+ taskDef.setFamily("logs-family");+ taskDef.setContainerDefinitions(List.of(main));++ EcsTask task = new EcsTask();+ task.setTaskArn(TASK_ARN);++ manager.startTask(task, taskDef, List.of(), "us-east-1");+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.javaindex e98ce3b292..d40ac4d8be 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerVolumesFromTest.java@@ -146,7 +146,9 @@ class EcsContainerManagerVolumesFromTest {ContainerDefinition router = definition("router", "router:latest");router.setFirelensConfiguration(new FirelensConfiguration("fluentbit", Map.of()));router.setVolumesFrom(List.of(new VolumeFrom("source", false)));+ router.setLogConfiguration(awslogs());ContainerDefinition source = definition("source", "sidecar:latest");+ source.setLogConfiguration(awslogs());EcsTask ecsTask = task();EcsTaskHandle handle = manager.startTask(@@ -267,6 +269,10 @@ class EcsContainerManagerVolumesFromTest {return definition;}+ private static LogConfiguration awslogs() {+ return new LogConfiguration("awslogs", Map.of("awslogs-group", "/ecs/volumesfrom-family"), null);+ }+private static TaskDefinition taskDefinition(List<ContainerDefinition> definitions) {TaskDefinition taskDefinition = new TaskDefinition();taskDefinition.setFamily("volumes-from-family");