Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ecs): retry FireLens configuration uploads (#6075)

Aniket Shukla17f32c9d047d
diff --git a/docs/services/ecs.md b/docs/services/ecs.md
index 8fe58cce89..ee1e9545a6 100644
--- a/docs/services/ecs.md
+++ b/docs/services/ecs.md
@@ -196,6 +196,11 @@ other case, the injected `FLUENT_HOST` (the router's container IP) must be reach
config is written to `/fluent-bit/etc/fluent-bit.conf`. Fluentd config is
written to `/fluentd/etc/fluent.conf` and uses `@type` (not `Name`) for output plugins; Floci
does not inject an `endpoint` into Fluentd outputs.
+FireLens configuration archive uploads retry transient Docker connection failures up to six
+attempts, with exponential backoff starting at 500 ms. Each attempt sends the complete prebuilt
+archive through a fresh byte-backed stream, without a streaming pipe or writer thread.
+The router and application containers start only after the upload succeeds;
+permanent errors or exhausted retries follow the existing task-launch cleanup path.
`config-file-type=s3` follows where ECS itself draws the line. `RegisterTaskDefinition` rejects
it for a Fargate-compatible task definition, with `Fargate launch type does not support
FirelensConfiguration config file from 's3'`, and rejects a `config-file-value` that is not an S3
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.java
index aec79c55ad..7046916776 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.java
@@ -62,8 +62,19 @@ public final class RetryingTarCopier {
public static void copyBytes(DockerClient docker, String containerId, String remotePath,
String entryName, byte[] content, int mode,
int maxAttempts, long backoffMillis) {
- byte[] tar = singleFileTar(entryName, content, mode);
- retry(entryName, containerId, maxAttempts, backoffMillis, () ->
+ copyArchive(docker, containerId, remotePath, entryName, singleFileTar(entryName, content, mode),
+ maxAttempts, backoffMillis);
+ }
+
+ /** Copies an already-built tar through fresh byte-backed streams; do not modify it during the copy. */
+ public static void copyArchive(DockerClient docker, String containerId, String remotePath,
+ String label, byte[] tar) {
+ copyArchive(docker, containerId, remotePath, label, tar, DEFAULT_MAX_ATTEMPTS, DEFAULT_BACKOFF_MS);
+ }
+
+ public static void copyArchive(DockerClient docker, String containerId, String remotePath,
+ String label, byte[] tar, int maxAttempts, long backoffMillis) {
+ retry(label, containerId, maxAttempts, backoffMillis, () ->
docker.copyArchiveToContainerCmd(containerId)
.withRemotePath(remotePath)
.withTarInputStream(new ByteArrayInputStream(tar))
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
index fa3f7e11d1..598ca07851 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
@@ -26,6 +26,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;
+import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;
import io.github.hectorvent.floci.services.ec2.Ec2Service;
import io.github.hectorvent.floci.services.ec2.SecurityGroupFirewallManager;
import io.github.hectorvent.floci.services.ec2.SecurityGroupNftCompiler;
@@ -70,7 +71,6 @@ import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.jboss.logging.Logger;
-import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
@@ -243,6 +243,11 @@ public class EcsContainerManager {
*/
public EcsTaskHandle startTask(EcsTask task, TaskDefinition taskDef,
List<ContainerOverride> containerOverrides, String region) {
+ return startTask(task, taskDef, containerOverrides, region, RetryingTarCopier.DEFAULT_BACKOFF_MS);
+ }
+
+ EcsTaskHandle startTask(EcsTask task, TaskDefinition taskDef,
+ List<ContainerOverride> containerOverrides, String region, long firelensBackoffMillis) {
String taskId = extractTaskId(task.getTaskArn());
Map<String, String> containerIds = new LinkedHashMap<>();
@@ -543,7 +548,7 @@ public class EcsContainerManager {
dockerId = lifecycleManager.create(spec);
try {
copyFirelensConfig(dockerId, firelensConfig, isFluentdRouter(firelensRouter),
- firelensExternalConfig);
+ firelensExternalConfig, firelensBackoffMillis);
lifecycleManager.startCreated(dockerId, spec);
} catch (RuntimeException e) {
lifecycleManager.removeIfExists(dockerId);
@@ -1784,7 +1789,7 @@ public class EcsContainerManager {
}
private void copyFirelensConfig(
- String containerId, String configText, boolean fluentd, String externalConfig) {
+ String containerId, String configText, boolean fluentd, String externalConfig, long backoffMillis) {
byte[] content = configText.getBytes(StandardCharsets.UTF_8);
String fileName = fluentd ? "fluent.conf" : "fluent-bit.conf";
String remotePath = fluentd ? "/fluentd/etc" : "/fluent-bit/etc";
@@ -1798,10 +1803,9 @@ public class EcsContainerManager {
throw new AwsException("ClientException",
"unable to write FireLens config: " + e.getMessage(), 500);
}
- lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)
- .withRemotePath(remotePath)
- .withTarInputStream(new ByteArrayInputStream(archive.toByteArray()))
- .exec();
+ byte[] tarBytes = archive.toByteArray();
+ RetryingTarCopier.copyArchive(lifecycleManager.getDockerClient(), containerId,
+ remotePath, "firelens-config", tarBytes, RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, backoffMillis);
}
private static void putTarEntry(TarArchiveOutputStream tar, String name, byte[] content) throws IOException {
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.java
index 272a45139c..2ac6f5d2e6 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.java
@@ -8,6 +8,7 @@ import com.github.dockerjava.core.DefaultDockerClientConfig;
import com.github.dockerjava.core.DockerClientImpl;
import com.github.dockerjava.core.util.CompressArchiveUtil;
import com.github.dockerjava.transport.DockerHttpClient;
+import io.github.hectorvent.floci.testing.DockerArchiveUploadFixture;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.junit.jupiter.api.Test;
@@ -33,6 +34,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotSame;
@@ -125,6 +127,60 @@ class RetryingTarCopierTest {
assertEquals(1, calls.get());
}
+ @ParameterizedTest
+ @ValueSource(ints = {0, 73, 8192})
+ void copyArchiveReplaysTheExactBytesWithoutAStreamingPipe(int consumedBytes) {
+ DockerClient docker = mock(DockerClient.class);
+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "c-1");
+ uploads.failures = 1;
+ uploads.consumedBytes = consumedBytes;
+ uploads.failure = brokenPipe();
+ byte[] archive = RetryingTarCopier.singleFileTar("config", new byte[16384], 0644);
+
+ RetryingTarCopier.copyArchive(docker, "c-1", "/etc", "config-archive", archive, MAX_ATTEMPTS, 0L);
+
+ assertEquals(2, uploads.commands.size());
+ assertEquals(1, uploads.completed.size());
+ assertArrayEquals(archive, uploads.completed.getFirst(), "an existing tar must not be wrapped in another tar");
+ assertNotSame(uploads.streams.getFirst(), uploads.streams.get(1));
+ for (int attempt = 0; attempt < uploads.commands.size(); attempt++) {
+ assertInstanceOf(ByteArrayInputStream.class, uploads.streams.get(attempt));
+ verify(uploads.commands.get(attempt)).withRemotePath("/etc");
+ }
+ }
+
+ @Test
+ void copyArchiveStopsAtTheRetryBudget() {
+ DockerClient docker = mock(DockerClient.class);
+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "c-1");
+ uploads.failures = Integer.MAX_VALUE;
+ uploads.failure = brokenPipe();
+
+ RuntimeException thrown = assertThrows(RuntimeException.class,
+ () -> RetryingTarCopier.copyArchive(docker, "c-1", "/etc", "config-archive",
+ new byte[1024], MAX_ATTEMPTS, 0L));
+
+ assertSame(uploads.failure, thrown.getCause());
+ assertTrue(thrown.getMessage().contains("config-archive"), thrown.getMessage());
+ assertEquals(MAX_ATTEMPTS, uploads.commands.size());
+ assertTrue(uploads.completed.isEmpty());
+ }
+
+ @Test
+ void copyArchiveDoesNotRetryDaemonRejection() {
+ DockerClient docker = mock(DockerClient.class);
+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "c-1");
+ uploads.failures = Integer.MAX_VALUE;
+ uploads.failure = new NotFoundException("container missing");
+
+ RuntimeException thrown = assertThrows(RuntimeException.class,
+ () -> RetryingTarCopier.copyArchive(docker, "c-1", "/etc", "config-archive",
+ new byte[1024], MAX_ATTEMPTS, 0L));
+
+ assertSame(uploads.failure, thrown.getCause());
+ assertEquals(1, uploads.commands.size());
+ }
+
// copyFile streams tarWriter through a pipe on its own thread; if the writer fails before
// producing any bytes (e.g. the source file vanished), closing the pipe with nothing written
// looks like a clean, empty tar to a reader that isn't checking for that, the daemon accepts
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java
index 5b7a7c7b45..633c91e506 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java
@@ -4,6 +4,7 @@ import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;
import com.github.dockerjava.api.command.InspectVolumeCmd;
import com.github.dockerjava.api.command.InspectVolumeResponse;
+import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.model.LogConfig;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
@@ -15,6 +16,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.C
import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;
+import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;
import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
import io.github.hectorvent.floci.services.ecs.model.EcsTask;
@@ -28,14 +30,21 @@ import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.ssm.SsmService;
+import io.github.hectorvent.floci.testing.DockerArchiveUploadFixture;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor;
import org.mockito.InOrder;
+import org.mockito.MockMakers;
+import org.mockito.MockedStatic;
import java.io.ByteArrayInputStream;
+import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
@@ -43,6 +52,10 @@ import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@@ -50,11 +63,14 @@ import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.RETURNS_SELF;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
class EcsContainerManagerFirelensTest {
@@ -64,6 +80,8 @@ class EcsContainerManagerFirelensTest {
private ContainerLogStreamer logStreamer;
private DockerClient dockerClient;
private CopyArchiveToContainerCmd copyCmd;
+ private InputStream uploadStream;
+ private byte[] uploadedArchive;
private EcsContainerManager manager;
@BeforeEach
@@ -89,6 +107,14 @@ class EcsContainerManagerFirelensTest {
when(inspectVolumeCmd.exec()).thenReturn(volume);
when(volume.getMountpoint()).thenReturn("/var/lib/docker/volumes/floci-ecs-firelens-abc123/_data");
copyCmd = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);
+ when(copyCmd.withTarInputStream(any(InputStream.class))).thenAnswer(invocation -> {
+ uploadStream = invocation.getArgument(0);
+ return copyCmd;
+ });
+ doAnswer(invocation -> {
+ uploadedArchive = uploadStream.readAllBytes();
+ return null;
+ }).when(copyCmd).exec();
when(dockerClient.copyArchiveToContainerCmd("router-id")).thenReturn(copyCmd);
logStreamer = mock(ContainerLogStreamer.class);
@@ -158,9 +184,8 @@ class EcsContainerManagerFirelensTest {
assertTrue(handle.getFirelensVolumeName().contains("firelens"));
verify(copyCmd).withRemotePath("/fluent-bit/etc");
- ArgumentCaptor<InputStream> configArchive = ArgumentCaptor.forClass(InputStream.class);
- verify(copyCmd).withTarInputStream(configArchive.capture());
- assertTrue(readFluentBitConf(configArchive.getValue()).contains(
+ verify(copyCmd).withTarInputStream(any(InputStream.class));
+ assertTrue(readFluentBitConf(new ByteArrayInputStream(uploadedArchive)).contains(
" Endpoint http://host.docker.internal:4566"));
}
@@ -201,9 +226,8 @@ class EcsContainerManagerFirelensTest {
logConfig.getValue().getConfig().get("fluentd-address"));
verify(copyCmd).withRemotePath("/fluentd/etc");
- ArgumentCaptor<InputStream> configArchive = ArgumentCaptor.forClass(InputStream.class);
- verify(copyCmd).withTarInputStream(configArchive.capture());
- String conf = readFluentBitConf(configArchive.getValue());
+ verify(copyCmd).withTarInputStream(any(InputStream.class));
+ String conf = readFluentBitConf(new ByteArrayInputStream(uploadedArchive));
assertTrue(conf.contains("@type unix"));
assertTrue(conf.contains("<match app-firelens**>"));
assertTrue(conf.contains("@type stdout"));
@@ -245,9 +269,8 @@ class EcsContainerManagerFirelensTest {
verify(s3Service).getObject("firelens-configs", "extra.conf");
verify(copyCmd).withRemotePath("/fluent-bit/etc");
- ArgumentCaptor<InputStream> configArchive = ArgumentCaptor.forClass(InputStream.class);
- verify(copyCmd).withTarInputStream(configArchive.capture());
- Map<String, String> files = readFirelensArchive(configArchive.getValue());
+ verify(copyCmd).withTarInputStream(any(InputStream.class));
+ Map<String, String> files = readFirelensArchive(new ByteArrayInputStream(uploadedArchive));
assertEquals(2, files.size(), files::toString);
assertTrue(files.get("fluent-bit.conf").contains(
"@INCLUDE /fluent-bit/etc/external.conf"), files.get("fluent-bit.conf"));
@@ -285,9 +308,8 @@ class EcsContainerManagerFirelensTest {
manager.startTask(task, taskDef, null, "us-east-1");
verify(copyCmd).withRemotePath("/fluentd/etc");
- ArgumentCaptor<InputStream> configArchive = ArgumentCaptor.forClass(InputStream.class);
- verify(copyCmd).withTarInputStream(configArchive.capture());
- Map<String, String> files = readFirelensArchive(configArchive.getValue());
+ verify(copyCmd).withTarInputStream(any(InputStream.class));
+ Map<String, String> files = readFirelensArchive(new ByteArrayInputStream(uploadedArchive));
assertTrue(files.get("fluent.conf").contains(
"@include /fluentd/etc/external.conf"), files.get("fluent.conf"));
assertTrue(files.containsKey("external.conf"), files::toString);
@@ -380,6 +402,180 @@ class EcsContainerManagerFirelensTest {
verify(lifecycleManager, never()).create(any(ContainerSpec.class));
}
+ @ParameterizedTest
+ @CsvSource({"true,500", "false,0", "false,13"})
+ void firelensUploadUsesTheSelectedBackoffAndSharedRetryBudget(boolean productionDefaults, long backoffMillis) {
+ try (MockedStatic<RetryingTarCopier> copies = mockStatic(RetryingTarCopier.class,
+ withSettings().mockMaker(MockMakers.INLINE))) {
+ TaskDefinition definition = retryTaskDefinition("fluentbit", false, "");
+ if (productionDefaults) {
+ manager.startTask(task(), definition, null, "us-east-1");
+ } else {
+ manager.startTask(task(), definition, null, "us-east-1", backoffMillis);
+ }
+
+ copies.verify(() -> RetryingTarCopier.copyArchive(eq(dockerClient), eq("router-id"),
+ eq("/fluent-bit/etc"), eq("firelens-config"), any(byte[].class),
+ eq(6), eq(backoffMillis)));
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"fluentbit,false", "fluentbit,true", "fluentd,false", "fluentd,true"})
+ void uploadsConfigFromMemoryWithoutAStreamingPipe(String type, boolean external) {
+ manager.startTask(task(), retryTaskDefinition(type, external, "# external config\n"), null, "us-east-1");
+
+ assertInstanceOf(ByteArrayInputStream.class, uploadStream,
+ "an already-built archive must not allocate a streaming pipe and writer thread");
+ }
+
+ @ParameterizedTest
+ @CsvSource({"fluentbit,false,0", "fluentbit,false,73", "fluentbit,false,8192",
+ "fluentbit,true,0", "fluentbit,true,73", "fluentbit,true,8192",
+ "fluentd,false,0", "fluentd,false,73", "fluentd,false,8192",
+ "fluentd,true,0", "fluentd,true,73", "fluentd,true,8192"})
+ void retriesConsumedConfigArchiveBeforeStartingContainers(String type, boolean external, int consumedBytes)
+ throws Exception {
+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(dockerClient, "router-id");
+ uploads.failures = 1;
+ uploads.consumedBytes = consumedBytes;
+ uploads.failure = new RuntimeException(new IOException("Connection reset"));
+ String extra = "# café ../not-an-entry\n".repeat(1024);
+
+ EcsTaskHandle handle = manager.startTask(task(), retryTaskDefinition(type, external, extra), null, "us-east-1", 0L);
+
+ assertEquals(List.of("log_router", "app"), handle.getContainerIds().keySet().stream().toList());
+ assertEquals(2, uploads.commands.size());
+ assertEquals(1, uploads.completed.size());
+ assertNotSame(uploads.streams.getFirst(), uploads.streams.get(1));
+ String path = "fluentd".equals(type) ? "/fluentd/etc" : "/fluent-bit/etc";
+ String file = "fluentd".equals(type) ? "fluent.conf" : "fluent-bit.conf";
+ verify(uploads.commands.get(1)).withRemotePath(path);
+ try (TarArchiveInputStream tar = new TarArchiveInputStream(new ByteArrayInputStream(uploads.completed.getFirst()))) {
+ assertConfigEntry(tar, file, expectedRetryConfig(type, external));
+ if (external) {
+ assertConfigEntry(tar, "external.conf", extra);
+ }
+ assertNull(tar.getNextEntry());
+ }
+ InOrder order = inOrder(lifecycleManager, uploads.commands.get(1));
+ order.verify(lifecycleManager).create(any());
+ order.verify(uploads.commands.get(1)).exec();
+ order.verify(lifecycleManager).startCreated(eq("router-id"), any());
+ order.verify(lifecycleManager).createAndStart(any());
+ verify(lifecycleManager, never()).removeIfExists(anyString());
+ if (external) {
+ verify(s3Service).getObject("firelens-configs", "extra.conf");
+ }
+ }
+
+ @Test
+ void exhaustedConfigUploadCleansUpBeforeAnyContainerStarts() {
+ DockerArchiveUploadFixture uploads = failingUploads();
+
+ RuntimeException failure = assertThrows(RuntimeException.class, () -> manager.startTask(
+ task(), retryTaskDefinition("fluentbit", true, "# external config\n"), null, "us-east-1", 0L));
+
+ assertSame(uploads.failure, failure.getCause());
+ assertEquals(6, uploads.commands.size());
+ assertEquals(0, uploads.completed.size());
+ verifyFailedLaunchCleanup();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"fluentbit", "fluentd"})
+ void daemonRejectionDoesNotRetryAndCleansUp(String type) {
+ DockerArchiveUploadFixture uploads = failingUploads();
+ uploads.failure = new NotFoundException("container missing");
+
+ RuntimeException failure = assertThrows(RuntimeException.class, () -> manager.startTask(
+ task(), retryTaskDefinition(type, false, ""), null, "us-east-1"));
+
+ assertSame(uploads.failure, failure.getCause());
+ assertEquals(1, uploads.commands.size());
+ verifyFailedLaunchCleanup();
+ }
+
+ @Test
+ void routerStartTransportFailureDoesNotRetryTheUploadOrStart() {
+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(dockerClient, "router-id");
+ RuntimeException transportFailure = new RuntimeException(new IOException("Connection reset"));
+ when(lifecycleManager.startCreated(anyString(), any())).thenThrow(transportFailure);
+
+ RuntimeException failure = assertThrows(RuntimeException.class, () -> manager.startTask(
+ task(), retryTaskDefinition("fluentbit", false, ""), null, "us-east-1"));
+
+ assertSame(transportFailure, failure);
+ assertEquals(1, uploads.commands.size());
+ assertEquals(1, uploads.completed.size());
+ verify(lifecycleManager).startCreated(eq("router-id"), any());
+ verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager).removeIfExists("router-id");
+ verifyFirelensVolumeRemoved();
+ }
+
+ private DockerArchiveUploadFixture failingUploads() {
+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(dockerClient, "router-id");
+ uploads.failures = Integer.MAX_VALUE;
+ uploads.consumedBytes = 73;
+ uploads.failure = new RuntimeException(new IOException("Connection reset"));
+ return uploads;
+ }
+
+ private void verifyFailedLaunchCleanup() {
+ verify(lifecycleManager).create(any());
+ verify(lifecycleManager, never()).startCreated(anyString(), any());
+ verify(lifecycleManager, never()).createAndStart(any());
+ verify(lifecycleManager).removeIfExists("router-id");
+ verifyFirelensVolumeRemoved();
+ }
+
+ private void verifyFirelensVolumeRemoved() {
+ ArgumentCaptor<String> volumeName = ArgumentCaptor.forClass(String.class);
+ verify(lifecycleManager).ensureVolume(volumeName.capture());
+ verify(lifecycleManager).removeVolume(volumeName.getValue());
+ }
+
+ private TaskDefinition retryTaskDefinition(String type, boolean external, String extra) {
+ ContainerDefinition router = router(type, List.of());
+ Map<String, String> options = external
+ ? Map.of("enable-ecs-log-metadata", "false", "config-file-type", "s3",
+ "config-file-value", "arn:aws:s3:::firelens-configs/extra.conf")
+ : Map.of("enable-ecs-log-metadata", "false");
+ router.setFirelensConfiguration(new FirelensConfiguration(type, options));
+ ContainerDefinition app = new ContainerDefinition();
+ app.setName("app");
+ app.setImage("app:latest");
+ app.setLogConfiguration(new LogConfiguration("awsfirelens",
+ Map.of("fluentd".equals(type) ? "@type" : "Name", "stdout"), null));
+ if (external) {
+ S3Object object = new S3Object();
+ object.setData(extra.getBytes(StandardCharsets.UTF_8));
+ when(s3Service.getObject("firelens-configs", "extra.conf")).thenReturn(object);
+ }
+ TaskDefinition definition = taskDefinition(List.of(app, router));
+ definition.setNetworkMode(NetworkMode.host);
+ return definition;
+ }
+
+ private static String expectedRetryConfig(String type, boolean external) {
+ if ("fluentd".equals(type)) {
+ return "<source>\n @type unix\n path /var/run/fluent.sock\n</source>\n\n"
+ + (external ? "@include /fluentd/etc/external.conf\n\n" : "")
+ + "<match app-firelens**>\n @type stdout\n</match>\n\n";
+ }
+ return "[INPUT]\n Name forward\n unix_path /var/run/fluent.sock\n Mem_Buf_Limit 25MB\n\n"
+ + (external ? "@INCLUDE /fluent-bit/etc/external.conf\n\n" : "")
+ + "[OUTPUT]\n Name stdout\n Match app-firelens*\n\n";
+ }
+
+ private static void assertConfigEntry(TarArchiveInputStream tar, String name, String content) throws IOException {
+ TarArchiveEntry entry = tar.getNextEntry();
+ assertEquals(name, entry.getName());
+ assertEquals(0100644, entry.getMode());
+ assertEquals(content, new String(tar.readAllBytes(), StandardCharsets.UTF_8));
+ }
+
private static ContainerDefinition router(String type, List<PortMapping> portMappings) {
ContainerDefinition router = new ContainerDefinition();
router.setName("log_router");