Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ecs): retry FireLens configuration uploads (#6075)
Aniket Shukla17f32c9d047d
diff --git a/docs/services/ecs.md b/docs/services/ecs.mdindex 8fe58cce89..ee1e9545a6 100644--- a/docs/services/ecs.md+++ b/docs/services/ecs.md@@ -196,6 +196,11 @@ other case, the injected `FLUENT_HOST` (the router's container IP) must be reachconfig is written to `/fluent-bit/etc/fluent-bit.conf`. Fluentd config iswritten to `/fluentd/etc/fluent.conf` and uses `@type` (not `Name`) for output plugins; Flocidoes not inject an `endpoint` into Fluentd outputs.+FireLens configuration archive uploads retry transient Docker connection failures up to six+attempts, with exponential backoff starting at 500 ms. Each attempt sends the complete prebuilt+archive through a fresh byte-backed stream, without a streaming pipe or writer thread.+The router and application containers start only after the upload succeeds;+permanent errors or exhausted retries follow the existing task-launch cleanup path.`config-file-type=s3` follows where ECS itself draws the line. `RegisterTaskDefinition` rejectsit for a Fargate-compatible task definition, with `Fargate launch type does not supportFirelensConfiguration config file from 's3'`, and rejects a `config-file-value` that is not an S3diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.javaindex aec79c55ad..7046916776 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopier.java@@ -62,8 +62,19 @@ public final class RetryingTarCopier {public static void copyBytes(DockerClient docker, String containerId, String remotePath,String entryName, byte[] content, int mode,int maxAttempts, long backoffMillis) {- byte[] tar = singleFileTar(entryName, content, mode);- retry(entryName, containerId, maxAttempts, backoffMillis, () ->+ copyArchive(docker, containerId, remotePath, entryName, singleFileTar(entryName, content, mode),+ maxAttempts, backoffMillis);+ }++ /** Copies an already-built tar through fresh byte-backed streams; do not modify it during the copy. */+ public static void copyArchive(DockerClient docker, String containerId, String remotePath,+ String label, byte[] tar) {+ copyArchive(docker, containerId, remotePath, label, tar, DEFAULT_MAX_ATTEMPTS, DEFAULT_BACKOFF_MS);+ }++ public static void copyArchive(DockerClient docker, String containerId, String remotePath,+ String label, byte[] tar, int maxAttempts, long backoffMillis) {+ retry(label, containerId, maxAttempts, backoffMillis, () ->docker.copyArchiveToContainerCmd(containerId).withRemotePath(remotePath).withTarInputStream(new ByteArrayInputStream(tar))diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.javaindex fa3f7e11d1..598ca07851 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java@@ -26,6 +26,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;+import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;import io.github.hectorvent.floci.services.ec2.Ec2Service;import io.github.hectorvent.floci.services.ec2.SecurityGroupFirewallManager;import io.github.hectorvent.floci.services.ec2.SecurityGroupNftCompiler;@@ -70,7 +71,6 @@ import org.apache.commons.compress.archivers.tar.TarArchiveEntry;import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;import org.jboss.logging.Logger;-import java.io.ByteArrayInputStream;import java.io.ByteArrayOutputStream;import java.io.Closeable;import java.io.IOException;@@ -243,6 +243,11 @@ public class EcsContainerManager {*/public EcsTaskHandle startTask(EcsTask task, TaskDefinition taskDef,List<ContainerOverride> containerOverrides, String region) {+ return startTask(task, taskDef, containerOverrides, region, RetryingTarCopier.DEFAULT_BACKOFF_MS);+ }++ EcsTaskHandle startTask(EcsTask task, TaskDefinition taskDef,+ List<ContainerOverride> containerOverrides, String region, long firelensBackoffMillis) {String taskId = extractTaskId(task.getTaskArn());Map<String, String> containerIds = new LinkedHashMap<>();@@ -543,7 +548,7 @@ public class EcsContainerManager {dockerId = lifecycleManager.create(spec);try {copyFirelensConfig(dockerId, firelensConfig, isFluentdRouter(firelensRouter),- firelensExternalConfig);+ firelensExternalConfig, firelensBackoffMillis);lifecycleManager.startCreated(dockerId, spec);} catch (RuntimeException e) {lifecycleManager.removeIfExists(dockerId);@@ -1784,7 +1789,7 @@ public class EcsContainerManager {}private void copyFirelensConfig(- String containerId, String configText, boolean fluentd, String externalConfig) {+ String containerId, String configText, boolean fluentd, String externalConfig, long backoffMillis) {byte[] content = configText.getBytes(StandardCharsets.UTF_8);String fileName = fluentd ? "fluent.conf" : "fluent-bit.conf";String remotePath = fluentd ? "/fluentd/etc" : "/fluent-bit/etc";@@ -1798,10 +1803,9 @@ public class EcsContainerManager {throw new AwsException("ClientException","unable to write FireLens config: " + e.getMessage(), 500);}- lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)- .withRemotePath(remotePath)- .withTarInputStream(new ByteArrayInputStream(archive.toByteArray()))- .exec();+ byte[] tarBytes = archive.toByteArray();+ RetryingTarCopier.copyArchive(lifecycleManager.getDockerClient(), containerId,+ remotePath, "firelens-config", tarBytes, RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, backoffMillis);}private static void putTarEntry(TarArchiveOutputStream tar, String name, byte[] content) throws IOException {diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.javaindex 272a45139c..2ac6f5d2e6 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/RetryingTarCopierTest.java@@ -8,6 +8,7 @@ import com.github.dockerjava.core.DefaultDockerClientConfig;import com.github.dockerjava.core.DockerClientImpl;import com.github.dockerjava.core.util.CompressArchiveUtil;import com.github.dockerjava.transport.DockerHttpClient;+import io.github.hectorvent.floci.testing.DockerArchiveUploadFixture;import org.apache.commons.compress.archivers.tar.TarArchiveEntry;import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;import org.junit.jupiter.api.Test;@@ -33,6 +34,7 @@ import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicInteger;import java.util.concurrent.atomic.AtomicReference;+import static org.junit.jupiter.api.Assertions.assertArrayEquals;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertInstanceOf;import static org.junit.jupiter.api.Assertions.assertNotSame;@@ -125,6 +127,60 @@ class RetryingTarCopierTest {assertEquals(1, calls.get());}+ @ParameterizedTest+ @ValueSource(ints = {0, 73, 8192})+ void copyArchiveReplaysTheExactBytesWithoutAStreamingPipe(int consumedBytes) {+ DockerClient docker = mock(DockerClient.class);+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "c-1");+ uploads.failures = 1;+ uploads.consumedBytes = consumedBytes;+ uploads.failure = brokenPipe();+ byte[] archive = RetryingTarCopier.singleFileTar("config", new byte[16384], 0644);++ RetryingTarCopier.copyArchive(docker, "c-1", "/etc", "config-archive", archive, MAX_ATTEMPTS, 0L);++ assertEquals(2, uploads.commands.size());+ assertEquals(1, uploads.completed.size());+ assertArrayEquals(archive, uploads.completed.getFirst(), "an existing tar must not be wrapped in another tar");+ assertNotSame(uploads.streams.getFirst(), uploads.streams.get(1));+ for (int attempt = 0; attempt < uploads.commands.size(); attempt++) {+ assertInstanceOf(ByteArrayInputStream.class, uploads.streams.get(attempt));+ verify(uploads.commands.get(attempt)).withRemotePath("/etc");+ }+ }++ @Test+ void copyArchiveStopsAtTheRetryBudget() {+ DockerClient docker = mock(DockerClient.class);+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "c-1");+ uploads.failures = Integer.MAX_VALUE;+ uploads.failure = brokenPipe();++ RuntimeException thrown = assertThrows(RuntimeException.class,+ () -> RetryingTarCopier.copyArchive(docker, "c-1", "/etc", "config-archive",+ new byte[1024], MAX_ATTEMPTS, 0L));++ assertSame(uploads.failure, thrown.getCause());+ assertTrue(thrown.getMessage().contains("config-archive"), thrown.getMessage());+ assertEquals(MAX_ATTEMPTS, uploads.commands.size());+ assertTrue(uploads.completed.isEmpty());+ }++ @Test+ void copyArchiveDoesNotRetryDaemonRejection() {+ DockerClient docker = mock(DockerClient.class);+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "c-1");+ uploads.failures = Integer.MAX_VALUE;+ uploads.failure = new NotFoundException("container missing");++ RuntimeException thrown = assertThrows(RuntimeException.class,+ () -> RetryingTarCopier.copyArchive(docker, "c-1", "/etc", "config-archive",+ new byte[1024], MAX_ATTEMPTS, 0L));++ assertSame(uploads.failure, thrown.getCause());+ assertEquals(1, uploads.commands.size());+ }+// copyFile streams tarWriter through a pipe on its own thread; if the writer fails before// producing any bytes (e.g. the source file vanished), closing the pipe with nothing written// looks like a clean, empty tar to a reader that isn't checking for that, the daemon acceptsdiff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.javaindex 5b7a7c7b45..633c91e506 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerFirelensTest.java@@ -4,6 +4,7 @@ import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;import com.github.dockerjava.api.command.InspectVolumeCmd;import com.github.dockerjava.api.command.InspectVolumeResponse;+import com.github.dockerjava.api.exception.NotFoundException;import com.github.dockerjava.api.model.LogConfig;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;@@ -15,6 +16,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.Cimport io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;+import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;import io.github.hectorvent.floci.services.ecs.model.EcsTask;@@ -28,14 +30,21 @@ import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.s3.model.S3Object;import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;import io.github.hectorvent.floci.services.ssm.SsmService;+import io.github.hectorvent.floci.testing.DockerArchiveUploadFixture;import org.apache.commons.compress.archivers.tar.TarArchiveEntry;import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;+import org.junit.jupiter.params.provider.ValueSource;import org.mockito.ArgumentCaptor;import org.mockito.InOrder;+import org.mockito.MockMakers;+import org.mockito.MockedStatic;import java.io.ByteArrayInputStream;+import java.io.IOException;import java.io.InputStream;import java.nio.charset.StandardCharsets;import java.util.LinkedHashMap;@@ -43,6 +52,10 @@ import java.util.List;import java.util.Map;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertInstanceOf;+import static org.junit.jupiter.api.Assertions.assertNotSame;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertSame;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;@@ -50,11 +63,14 @@ import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.RETURNS_DEEP_STUBS;import static org.mockito.Mockito.RETURNS_SELF;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.inOrder;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.mockStatic;import static org.mockito.Mockito.never;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;+import static org.mockito.Mockito.withSettings;class EcsContainerManagerFirelensTest {@@ -64,6 +80,8 @@ class EcsContainerManagerFirelensTest {private ContainerLogStreamer logStreamer;private DockerClient dockerClient;private CopyArchiveToContainerCmd copyCmd;+ private InputStream uploadStream;+ private byte[] uploadedArchive;private EcsContainerManager manager;@BeforeEach@@ -89,6 +107,14 @@ class EcsContainerManagerFirelensTest {when(inspectVolumeCmd.exec()).thenReturn(volume);when(volume.getMountpoint()).thenReturn("/var/lib/docker/volumes/floci-ecs-firelens-abc123/_data");copyCmd = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);+ when(copyCmd.withTarInputStream(any(InputStream.class))).thenAnswer(invocation -> {+ uploadStream = invocation.getArgument(0);+ return copyCmd;+ });+ doAnswer(invocation -> {+ uploadedArchive = uploadStream.readAllBytes();+ return null;+ }).when(copyCmd).exec();when(dockerClient.copyArchiveToContainerCmd("router-id")).thenReturn(copyCmd);logStreamer = mock(ContainerLogStreamer.class);@@ -158,9 +184,8 @@ class EcsContainerManagerFirelensTest {assertTrue(handle.getFirelensVolumeName().contains("firelens"));verify(copyCmd).withRemotePath("/fluent-bit/etc");- ArgumentCaptor<InputStream> configArchive = ArgumentCaptor.forClass(InputStream.class);- verify(copyCmd).withTarInputStream(configArchive.capture());- assertTrue(readFluentBitConf(configArchive.getValue()).contains(+ verify(copyCmd).withTarInputStream(any(InputStream.class));+ assertTrue(readFluentBitConf(new ByteArrayInputStream(uploadedArchive)).contains(" Endpoint http://host.docker.internal:4566"));}@@ -201,9 +226,8 @@ class EcsContainerManagerFirelensTest {logConfig.getValue().getConfig().get("fluentd-address"));verify(copyCmd).withRemotePath("/fluentd/etc");- ArgumentCaptor<InputStream> configArchive = ArgumentCaptor.forClass(InputStream.class);- verify(copyCmd).withTarInputStream(configArchive.capture());- String conf = readFluentBitConf(configArchive.getValue());+ verify(copyCmd).withTarInputStream(any(InputStream.class));+ String conf = readFluentBitConf(new ByteArrayInputStream(uploadedArchive));assertTrue(conf.contains("@type unix"));assertTrue(conf.contains("<match app-firelens**>"));assertTrue(conf.contains("@type stdout"));@@ -245,9 +269,8 @@ class EcsContainerManagerFirelensTest {verify(s3Service).getObject("firelens-configs", "extra.conf");verify(copyCmd).withRemotePath("/fluent-bit/etc");- ArgumentCaptor<InputStream> configArchive = ArgumentCaptor.forClass(InputStream.class);- verify(copyCmd).withTarInputStream(configArchive.capture());- Map<String, String> files = readFirelensArchive(configArchive.getValue());+ verify(copyCmd).withTarInputStream(any(InputStream.class));+ Map<String, String> files = readFirelensArchive(new ByteArrayInputStream(uploadedArchive));assertEquals(2, files.size(), files::toString);assertTrue(files.get("fluent-bit.conf").contains("@INCLUDE /fluent-bit/etc/external.conf"), files.get("fluent-bit.conf"));@@ -285,9 +308,8 @@ class EcsContainerManagerFirelensTest {manager.startTask(task, taskDef, null, "us-east-1");verify(copyCmd).withRemotePath("/fluentd/etc");- ArgumentCaptor<InputStream> configArchive = ArgumentCaptor.forClass(InputStream.class);- verify(copyCmd).withTarInputStream(configArchive.capture());- Map<String, String> files = readFirelensArchive(configArchive.getValue());+ verify(copyCmd).withTarInputStream(any(InputStream.class));+ Map<String, String> files = readFirelensArchive(new ByteArrayInputStream(uploadedArchive));assertTrue(files.get("fluent.conf").contains("@include /fluentd/etc/external.conf"), files.get("fluent.conf"));assertTrue(files.containsKey("external.conf"), files::toString);@@ -380,6 +402,180 @@ class EcsContainerManagerFirelensTest {verify(lifecycleManager, never()).create(any(ContainerSpec.class));}+ @ParameterizedTest+ @CsvSource({"true,500", "false,0", "false,13"})+ void firelensUploadUsesTheSelectedBackoffAndSharedRetryBudget(boolean productionDefaults, long backoffMillis) {+ try (MockedStatic<RetryingTarCopier> copies = mockStatic(RetryingTarCopier.class,+ withSettings().mockMaker(MockMakers.INLINE))) {+ TaskDefinition definition = retryTaskDefinition("fluentbit", false, "");+ if (productionDefaults) {+ manager.startTask(task(), definition, null, "us-east-1");+ } else {+ manager.startTask(task(), definition, null, "us-east-1", backoffMillis);+ }++ copies.verify(() -> RetryingTarCopier.copyArchive(eq(dockerClient), eq("router-id"),+ eq("/fluent-bit/etc"), eq("firelens-config"), any(byte[].class),+ eq(6), eq(backoffMillis)));+ }+ }++ @ParameterizedTest+ @CsvSource({"fluentbit,false", "fluentbit,true", "fluentd,false", "fluentd,true"})+ void uploadsConfigFromMemoryWithoutAStreamingPipe(String type, boolean external) {+ manager.startTask(task(), retryTaskDefinition(type, external, "# external config\n"), null, "us-east-1");++ assertInstanceOf(ByteArrayInputStream.class, uploadStream,+ "an already-built archive must not allocate a streaming pipe and writer thread");+ }++ @ParameterizedTest+ @CsvSource({"fluentbit,false,0", "fluentbit,false,73", "fluentbit,false,8192",+ "fluentbit,true,0", "fluentbit,true,73", "fluentbit,true,8192",+ "fluentd,false,0", "fluentd,false,73", "fluentd,false,8192",+ "fluentd,true,0", "fluentd,true,73", "fluentd,true,8192"})+ void retriesConsumedConfigArchiveBeforeStartingContainers(String type, boolean external, int consumedBytes)+ throws Exception {+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(dockerClient, "router-id");+ uploads.failures = 1;+ uploads.consumedBytes = consumedBytes;+ uploads.failure = new RuntimeException(new IOException("Connection reset"));+ String extra = "# café ../not-an-entry\n".repeat(1024);++ EcsTaskHandle handle = manager.startTask(task(), retryTaskDefinition(type, external, extra), null, "us-east-1", 0L);++ assertEquals(List.of("log_router", "app"), handle.getContainerIds().keySet().stream().toList());+ assertEquals(2, uploads.commands.size());+ assertEquals(1, uploads.completed.size());+ assertNotSame(uploads.streams.getFirst(), uploads.streams.get(1));+ String path = "fluentd".equals(type) ? "/fluentd/etc" : "/fluent-bit/etc";+ String file = "fluentd".equals(type) ? "fluent.conf" : "fluent-bit.conf";+ verify(uploads.commands.get(1)).withRemotePath(path);+ try (TarArchiveInputStream tar = new TarArchiveInputStream(new ByteArrayInputStream(uploads.completed.getFirst()))) {+ assertConfigEntry(tar, file, expectedRetryConfig(type, external));+ if (external) {+ assertConfigEntry(tar, "external.conf", extra);+ }+ assertNull(tar.getNextEntry());+ }+ InOrder order = inOrder(lifecycleManager, uploads.commands.get(1));+ order.verify(lifecycleManager).create(any());+ order.verify(uploads.commands.get(1)).exec();+ order.verify(lifecycleManager).startCreated(eq("router-id"), any());+ order.verify(lifecycleManager).createAndStart(any());+ verify(lifecycleManager, never()).removeIfExists(anyString());+ if (external) {+ verify(s3Service).getObject("firelens-configs", "extra.conf");+ }+ }++ @Test+ void exhaustedConfigUploadCleansUpBeforeAnyContainerStarts() {+ DockerArchiveUploadFixture uploads = failingUploads();++ RuntimeException failure = assertThrows(RuntimeException.class, () -> manager.startTask(+ task(), retryTaskDefinition("fluentbit", true, "# external config\n"), null, "us-east-1", 0L));++ assertSame(uploads.failure, failure.getCause());+ assertEquals(6, uploads.commands.size());+ assertEquals(0, uploads.completed.size());+ verifyFailedLaunchCleanup();+ }++ @ParameterizedTest+ @ValueSource(strings = {"fluentbit", "fluentd"})+ void daemonRejectionDoesNotRetryAndCleansUp(String type) {+ DockerArchiveUploadFixture uploads = failingUploads();+ uploads.failure = new NotFoundException("container missing");++ RuntimeException failure = assertThrows(RuntimeException.class, () -> manager.startTask(+ task(), retryTaskDefinition(type, false, ""), null, "us-east-1"));++ assertSame(uploads.failure, failure.getCause());+ assertEquals(1, uploads.commands.size());+ verifyFailedLaunchCleanup();+ }++ @Test+ void routerStartTransportFailureDoesNotRetryTheUploadOrStart() {+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(dockerClient, "router-id");+ RuntimeException transportFailure = new RuntimeException(new IOException("Connection reset"));+ when(lifecycleManager.startCreated(anyString(), any())).thenThrow(transportFailure);++ RuntimeException failure = assertThrows(RuntimeException.class, () -> manager.startTask(+ task(), retryTaskDefinition("fluentbit", false, ""), null, "us-east-1"));++ assertSame(transportFailure, failure);+ assertEquals(1, uploads.commands.size());+ assertEquals(1, uploads.completed.size());+ verify(lifecycleManager).startCreated(eq("router-id"), any());+ verify(lifecycleManager, never()).createAndStart(any());+ verify(lifecycleManager).removeIfExists("router-id");+ verifyFirelensVolumeRemoved();+ }++ private DockerArchiveUploadFixture failingUploads() {+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(dockerClient, "router-id");+ uploads.failures = Integer.MAX_VALUE;+ uploads.consumedBytes = 73;+ uploads.failure = new RuntimeException(new IOException("Connection reset"));+ return uploads;+ }++ private void verifyFailedLaunchCleanup() {+ verify(lifecycleManager).create(any());+ verify(lifecycleManager, never()).startCreated(anyString(), any());+ verify(lifecycleManager, never()).createAndStart(any());+ verify(lifecycleManager).removeIfExists("router-id");+ verifyFirelensVolumeRemoved();+ }++ private void verifyFirelensVolumeRemoved() {+ ArgumentCaptor<String> volumeName = ArgumentCaptor.forClass(String.class);+ verify(lifecycleManager).ensureVolume(volumeName.capture());+ verify(lifecycleManager).removeVolume(volumeName.getValue());+ }++ private TaskDefinition retryTaskDefinition(String type, boolean external, String extra) {+ ContainerDefinition router = router(type, List.of());+ Map<String, String> options = external+ ? Map.of("enable-ecs-log-metadata", "false", "config-file-type", "s3",+ "config-file-value", "arn:aws:s3:::firelens-configs/extra.conf")+ : Map.of("enable-ecs-log-metadata", "false");+ router.setFirelensConfiguration(new FirelensConfiguration(type, options));+ ContainerDefinition app = new ContainerDefinition();+ app.setName("app");+ app.setImage("app:latest");+ app.setLogConfiguration(new LogConfiguration("awsfirelens",+ Map.of("fluentd".equals(type) ? "@type" : "Name", "stdout"), null));+ if (external) {+ S3Object object = new S3Object();+ object.setData(extra.getBytes(StandardCharsets.UTF_8));+ when(s3Service.getObject("firelens-configs", "extra.conf")).thenReturn(object);+ }+ TaskDefinition definition = taskDefinition(List.of(app, router));+ definition.setNetworkMode(NetworkMode.host);+ return definition;+ }++ private static String expectedRetryConfig(String type, boolean external) {+ if ("fluentd".equals(type)) {+ return "<source>\n @type unix\n path /var/run/fluent.sock\n</source>\n\n"+ + (external ? "@include /fluentd/etc/external.conf\n\n" : "")+ + "<match app-firelens**>\n @type stdout\n</match>\n\n";+ }+ return "[INPUT]\n Name forward\n unix_path /var/run/fluent.sock\n Mem_Buf_Limit 25MB\n\n"+ + (external ? "@INCLUDE /fluent-bit/etc/external.conf\n\n" : "")+ + "[OUTPUT]\n Name stdout\n Match app-firelens*\n\n";+ }++ private static void assertConfigEntry(TarArchiveInputStream tar, String name, String content) throws IOException {+ TarArchiveEntry entry = tar.getNextEntry();+ assertEquals(name, entry.getName());+ assertEquals(0100644, entry.getMode());+ assertEquals(content, new String(tar.readAllBytes(), StandardCharsets.UTF_8));+ }+private static ContainerDefinition router(String type, List<PortMapping> portMappings) {ContainerDefinition router = new ContainerDefinition();router.setName("log_router");