Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(docker): follow adopted sidecar containers from now instead of replaying their log history (#3612)

Chris Jovanovskiebb79ed3c200
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java
index 55241f2ebe..b405e8542b 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java
@@ -4,6 +4,7 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsService;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.async.ResultCallback;
+import com.github.dockerjava.api.command.LogContainerCmd;
import com.github.dockerjava.api.model.Frame;
import com.github.dockerjava.api.model.StreamType;
import jakarta.enterprise.context.ApplicationScoped;
@@ -14,6 +15,7 @@ import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
+import java.time.Instant;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayDeque;
@@ -69,9 +71,27 @@ public class ContainerLogStreamer {
null, containerId, logGroup, logStream, region, logPrefix);
}
+ /**
+ * Like {@link #attach}, but only forwards lines the container emits from now on. Use it for a
+ * container that already existed before this process started: following from the beginning
+ * replays its whole history (hours of registry access logs after a busy day) into the console
+ * and CloudWatch Logs, where it evicts the current Lambda logs.
+ */
+ public Closeable attachFromNow(String containerId, String logGroup, String logStream,
+ String region, String logPrefix) {
+ return attachForAccount(null, containerId, logGroup, logStream, region, logPrefix, Instant.now());
+ }
+
public Closeable attachForAccount(
String accountId, String containerId, String logGroup, String logStream,
String region, String logPrefix) {
+ return attachForAccount(accountId, containerId, logGroup, logStream, region, logPrefix, null);
+ }
+
+ /** {@code since} of null follows the container's complete log history. */
+ Closeable attachForAccount(
+ String accountId, String containerId, String logGroup, String logStream,
+ String region, String logPrefix, Instant since) {
ensureLogGroupAndStreamForAccount(accountId, logGroup, logStream, region);
// Trim trailing whitespace and drop blank lines, then fan each reassembled line out to the
@@ -86,12 +106,15 @@ public class ContainerLogStreamer {
try {
LogReassemblyCallback callback = new LogReassemblyCallback(emitter);
- dockerClient.logContainerCmd(containerId)
+ LogContainerCmd command = dockerClient.logContainerCmd(containerId)
.withStdOut(true)
.withStdErr(true)
.withFollowStream(true)
- .withTimestamps(false)
- .exec(callback);
+ .withTimestamps(false);
+ if (since != null) {
+ command = command.withSince((int) since.getEpochSecond());
+ }
+ command.exec(callback);
return new ContainerLogHandle(callback);
} catch (Exception e) {
LOG.warnv("Could not attach log stream for container {0}: {1}", containerId, e.getMessage());
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java
index 224e7e6196..48c52c9566 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java
@@ -32,6 +32,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.TimeUnit;
/**
@@ -292,7 +293,7 @@ public class EcrRegistryManager {
LOG.infov("Started ECR backing registry {0} on host port {1}", name, String.valueOf(chosenPort));
// Attach log streaming (new feature)
- attachLogStream();
+ attachLogStream(false);
} catch (Exception e) {
// Release the reserved port unless the container actually started, so a
// failed start (e.g. Docker unreachable) does not permanently exhaust the
@@ -330,15 +331,17 @@ public class EcrRegistryManager {
specBuilder.withBind(hostDataPath, "/var/lib/registry");
}
- private void attachLogStream() {
+ // An adopted container carries history from before this process; only its new lines are wanted.
+ private void attachLogStream(boolean adopted) {
closeLogStream();
String shortId = containerId.length() >= 8 ? containerId.substring(0, 8) : containerId;
String logGroup = "/aws/ecr/registry";
String logStreamName = logStreamer.generateLogStreamName(shortId);
String region = regionResolver.getDefaultRegion();
- this.logStream = logStreamer.attach(
- containerId, logGroup, logStreamName, region, "ecr:registry");
+ this.logStream = adopted
+ ? logStreamer.attachFromNow(containerId, logGroup, logStreamName, region, "ecr:registry")
+ : logStreamer.attach(containerId, logGroup, logStreamName, region, "ecr:registry");
}
private void closeLogStream() {
@@ -354,15 +357,15 @@ public class EcrRegistryManager {
}
}
- private java.util.Optional<String> resolveRegistryDockerNetwork() {
- java.util.Optional<String> configured = config.services().ecr().dockerNetwork();
+ private Optional<String> resolveRegistryDockerNetwork() {
+ Optional<String> configured = config.services().ecr().dockerNetwork();
if (configured.isPresent() && !configured.get().isBlank()) {
return configured;
}
if (containerDetector.isRunningInContainer()) {
return currentContainerNetworkResolver.resolveNetworkName();
}
- return java.util.Optional.empty();
+ return Optional.empty();
}
private void runReconcileOnce() {
@@ -575,7 +578,7 @@ public class EcrRegistryManager {
containerId, String.valueOf(hostPort));
// Attach log streaming to adopted container
- attachLogStream();
+ attachLogStream(true);
} catch (Exception e) {
LOG.warnv("Failed to adopt existing ECR registry container: {0}", e.getMessage());
this.containerId = null;
diff --git a/src/main/java/io/github/hectorvent/floci/services/floci/ui/FlociUiManager.java b/src/main/java/io/github/hectorvent/floci/services/floci/ui/FlociUiManager.java
index f6159f2cef..b420a2c470 100644
--- a/src/main/java/io/github/hectorvent/floci/services/floci/ui/FlociUiManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/floci/ui/FlociUiManager.java
@@ -207,7 +207,7 @@ public class FlociUiManager {
this.lastError = null;
LOG.infov("Started web console sidecar {0} from {1} on host port {2}",
name, image, String.valueOf(hostPort));
- attachLogStream();
+ attachLogStream(false);
} catch (IllegalStateException e) {
// replaceIfEndpointDrifted() records its own specific message for a container with an
// existing sidecar to adopt, but a fresh start (no existing container) reaches this
@@ -746,20 +746,23 @@ public class FlociUiManager {
this.lastError = null;
LOG.infov("Adopted existing web console sidecar {0} on host port {1}",
containerId, String.valueOf(hostPort));
- attachLogStream();
+ attachLogStream(true);
} catch (Exception e) {
LOG.warnv("Failed to adopt the existing web console sidecar: {0}", e.getMessage());
this.containerId = null;
}
}
- private void attachLogStream() {
+ // An adopted container carries history from before this process; only its new lines are wanted.
+ private void attachLogStream(boolean adopted) {
closeLogStream();
String shortId = containerId.length() >= 8 ? containerId.substring(0, 8) : containerId;
String logGroup = "/floci/ui";
String logStreamName = logStreamer.generateLogStreamName(shortId);
String region = regionResolver.getDefaultRegion();
- this.logStream = logStreamer.attach(containerId, logGroup, logStreamName, region, "floci:ui");
+ this.logStream = adopted
+ ? logStreamer.attachFromNow(containerId, logGroup, logStreamName, region, "floci:ui")
+ : logStreamer.attach(containerId, logGroup, logStreamName, region, "floci:ui");
}
/** Releases the previous follower, so a restarted sidecar does not leave one behind. */
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java
index 04a76307cd..d4fbfed898 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java
@@ -1,11 +1,16 @@
package io.github.hectorvent.floci.core.common.docker;
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.LogContainerCmd;
import com.github.dockerjava.api.model.Frame;
import com.github.dockerjava.api.model.StreamType;
+import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsService;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
import java.io.Closeable;
import java.nio.charset.StandardCharsets;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -15,6 +20,13 @@ import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.RETURNS_SELF;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
class ContainerLogStreamerTest {
@@ -325,4 +337,36 @@ class ContainerLogStreamerTest {
}
assertEquals(List.of("unterminated-tail"), emitted);
}
+
+ private static LogContainerCmd attachWith(java.util.function.BiFunction<ContainerLogStreamer, String, Closeable> attach) {
+ DockerClient dockerClient = mock(DockerClient.class);
+ LogContainerCmd command = mock(LogContainerCmd.class, RETURNS_SELF);
+ when(dockerClient.logContainerCmd("container-1")).thenReturn(command);
+ ContainerLogStreamer streamer = new ContainerLogStreamer(dockerClient, mock(CloudWatchLogsService.class));
+ attach.apply(streamer, "container-1");
+ return command;
+ }
+
+ @Test
+ void attachFollowsTheCompleteHistoryByDefault() {
+ LogContainerCmd command = attachWith((streamer, id) ->
+ streamer.attach(id, "/aws/lambda/fn", "stream", "us-east-1", "lambda:fn"));
+
+ verify(command).withFollowStream(true);
+ verify(command, never()).withSince(anyInt());
+ }
+
+ @Test
+ void attachFromNowOnlyFollowsLinesEmittedAfterAttachment() {
+ long before = Instant.now().getEpochSecond();
+ LogContainerCmd command = attachWith((streamer, id) ->
+ streamer.attachFromNow(id, "/aws/ecr/registry", "stream", "us-east-1", "ecr:registry"));
+ long after = Instant.now().getEpochSecond();
+
+ ArgumentCaptor<Integer> since = ArgumentCaptor.forClass(Integer.class);
+ verify(command).withSince(since.capture());
+ assertTrue(since.getValue() >= before && since.getValue() <= after,
+ "since must be the attach time so an adopted container's history is not replayed");
+ verify(command).withFollowStream(true);
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManagerTest.java
index 11df9bca1c..8d48f7a754 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManagerTest.java
@@ -42,6 +42,7 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -248,6 +249,36 @@ class EcrRegistryManagerTest {
assertEquals("http://" + REGISTRY_NAME + ":5000", manager.httpClient().baseUrl());
}
+ @Test
+ void freshlyCreatedRegistryStreamsItsCompleteLog() {
+ when(lifecycleManager.createAndStart(any())).thenReturn(
+ new ContainerLifecycleManager.ContainerInfo("container-id", Map.of()));
+
+ manager.ensureStarted();
+
+ verify(logStreamer).attach(eq("container-id"), eq("/aws/ecr/registry"), any(), eq("us-east-1"), eq("ecr:registry"));
+ verify(logStreamer, never()).attachFromNow(any(), any(), any(), any(), any());
+ }
+
+ @Test
+ void adoptedRegistryStreamsOnlyNewLinesInsteadOfReplayingItsHistory() {
+ Container existing = Mockito.mock(Container.class);
+ when(existing.getId()).thenReturn("0123456789abcdef");
+ when(existing.getPorts()).thenReturn(new ContainerPort[] {
+ new ContainerPort().withIp("127.0.0.1").withPrivatePort(5000).withPublicPort(BASE_PORT + 1)
+ });
+ when(lifecycleManager.findByName(REGISTRY_NAME)).thenReturn(Optional.of(existing));
+ when(lifecycleManager.adopt("0123456789abcdef", List.of(5000)))
+ .thenReturn(new ContainerLifecycleManager.ContainerInfo("0123456789abcdef",
+ Map.of(5000, new ContainerLifecycleManager.EndpointInfo("172.17.0.5", 5000)),
+ Map.of(5000, BASE_PORT + 1)));
+
+ manager.ensureStarted();
+
+ verify(logStreamer).attachFromNow(eq("0123456789abcdef"), eq("/aws/ecr/registry"), any(), eq("us-east-1"), eq("ecr:registry"));
+ verify(logStreamer, never()).attach(any(), any(), any(), any(), any());
+ }
+
@Test
void adoptTracksPrivateBackingPortWhenRunningInsideDocker() {
// In container mode httpClient() uses the registry's in-network port. The adopted