Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): reassemble container log lines split across Docker frames (#1953)

Aniket Shukla6344852540ce
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index 65844ff352..42a35c8a89 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -164,34 +164,46 @@ public class ContainerLifecycleManager {
public void stopAndRemove(String containerId, Closeable logStream) {
LOG.infov("Stopping container {0}", containerId);
- // Close log stream first
- if (logStream != null) {
- try {
- logStream.close();
- } catch (Exception e) {
- LOG.debugv("Error closing log stream: {0}", e.getMessage());
- }
- }
-
- // Stop container
+ boolean stoppedOrMissing = false;
try {
dockerClient.stopContainerCmd(containerId).withTimeout(5).exec();
+ stoppedOrMissing = true;
} catch (NotFoundException e) {
LOG.debugv("Container {0} not found (already removed)", containerId);
- return;
+ stoppedOrMissing = true;
} catch (Exception e) {
LOG.warnv("Error stopping container {0}: {1}", containerId, e.getMessage());
}
- // Remove container
+ // Force removal is the recovery path when Docker could not stop the container cleanly. It also
+ // terminates the follow-log transport, allowing its terminal callback to drain the final tail.
+ boolean removedOrMissing = false;
try {
dockerClient.removeContainerCmd(containerId).withForce(true).exec();
+ removedOrMissing = true;
LOG.debugv("Removed container {0}", containerId);
} catch (NotFoundException e) {
// Already gone
+ removedOrMissing = true;
} catch (Exception e) {
LOG.warnv("Error removing container {0}: {1}", containerId, e.getMessage());
}
+
+ if (logStream != null && (stoppedOrMissing || removedOrMissing)) {
+ closeLogStreamAfterContainerStop(logStream);
+ }
+ }
+
+ /**
+ * Releases lifecycle ownership of a log stream after its container has stopped. Docker's terminal
+ * callback normally flushes the final tail; a bounded fallback handles a broken transport.
+ */
+ public void closeLogStreamAfterContainerStop(Closeable logStream) {
+ try {
+ logStream.close();
+ } catch (Exception e) {
+ LOG.debugv("Error closing log stream: {0}", e.getMessage());
+ }
}
/**
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java
index 0996536ff3..0fca8b72a6 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java
@@ -5,17 +5,32 @@ import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsService
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.async.ResultCallback;
import com.github.dockerjava.api.model.Frame;
+import com.github.dockerjava.api.model.StreamType;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.io.ByteArrayOutputStream;
import java.io.Closeable;
+import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Consumer;
/**
* Streams Docker container logs to both the Floci console logger and CloudWatch Logs.
@@ -38,26 +53,39 @@ public class ContainerLogStreamer {
/**
* Attaches a log stream to a container and forwards logs to CloudWatch Logs.
- * Returns a Closeable handle that should be closed when the container is stopped.
+ * Returns a lifecycle handle that should be closed after the container stops. Closing the handle
+ * allows Docker's reader a bounded drain period before a non-blocking fallback flushes the tail.
*
* @param containerId Docker container ID
* @param logGroup CloudWatch log group name (e.g., "/aws/lambda/myFunction")
* @param logStream CloudWatch log stream name (e.g., "2024/01/15/[$LATEST]abc123")
* @param region AWS region for CloudWatch Logs
* @param logPrefix prefix for console logging (e.g., "lambda:myFunction")
- * @return Closeable handle to stop the log stream
+ * @return Closeable lifecycle handle for the log stream
*/
public Closeable attach(String containerId, String logGroup, String logStream,
String region, String logPrefix) {
ensureLogGroupAndStream(logGroup, logStream, region);
+ // Trim trailing whitespace and drop blank lines, then fan each reassembled line out to the
+ // console logger and CloudWatch Logs.
+ Consumer<String> emitter = line -> {
+ String trimmed = line.stripTrailing();
+ if (!trimmed.isEmpty()) {
+ LOG.infov("[{0}] {1}", logPrefix, trimmed);
+ forwardToCloudWatchLogs(logGroup, logStream, region, trimmed);
+ }
+ };
+
try {
- return dockerClient.logContainerCmd(containerId)
+ LogReassemblyCallback callback = new LogReassemblyCallback(emitter);
+ dockerClient.logContainerCmd(containerId)
.withStdOut(true)
.withStdErr(true)
.withFollowStream(true)
.withTimestamps(false)
- .exec(frameCallback(logGroup, logStream, region, logPrefix));
+ .exec(callback);
+ return new ContainerLogHandle(callback);
} catch (Exception e) {
LOG.warnv("Could not attach log stream for container {0}: {1}", containerId, e.getMessage());
return null;
@@ -149,4 +177,312 @@ public class ContainerLogStreamer {
LOG.debugv("Could not forward log line to CloudWatch Logs: {0}", e.getMessage());
}
}
+
+ /**
+ * Docker frame callback that reassembles multiplexed stdout/stderr frames into newline-delimited
+ * lines and forwards each to {@code lineConsumer}. One logical line can span several frames and one
+ * frame can carry several lines, so bytes are buffered per stream and emitted one line at a time
+ * (matching real AWS Lambda).
+ */
+ static final class LogReassemblyCallback extends ResultCallback.Adapter<Frame> {
+
+ private static final ScheduledExecutorService CLOSE_DRAIN_SCHEDULER =
+ Executors.newSingleThreadScheduledExecutor(runnable -> {
+ Thread thread = new Thread(runnable, "floci-container-log-close");
+ thread.setDaemon(true);
+ return thread;
+ });
+ private static final long DEFAULT_CLOSE_GRACE_MILLIS = 2_000;
+
+ private final Map<StreamType, LogLineBuffer> buffers = new EnumMap<>(StreamType.class);
+ private final Map<StreamType, Long> tailSequences = new EnumMap<>(StreamType.class);
+ private final Deque<String> pendingLines = new ArrayDeque<>();
+ private final Object emissionLock = new Object();
+ // A fair write lock makes fallback finalization wait for callbacks already in progress, then blocks
+ // any later callback until the transport has been closed and the final tail has been flushed.
+ private final ReentrantReadWriteLock callbackGate = new ReentrantReadWriteLock(true);
+ private final Consumer<String> lineConsumer;
+ private final long closeGraceMillis;
+ // Guarded by 'buffers'. Set only by the Docker transport terminal callback; afterwards onNext
+ // refuses to append, so nothing is ever buffered past the final flush.
+ private boolean terminated;
+ // Guarded by 'buffers'. The most recent callback with an unfinished tail gets the largest value.
+ private long nextTailSequence;
+ // Guarded by 'buffers'. Cancels when Docker supplies its terminal callback before the fallback.
+ private ScheduledFuture<?> fallbackFinalization;
+ // Guarded by 'emissionLock'. Exactly one caller drains pendingLines at a time, preserving Docker
+ // callback order even when a terminal callback and onNext run on different threads.
+ private boolean emitting;
+
+ LogReassemblyCallback(Consumer<String> lineConsumer) {
+ this(lineConsumer, DEFAULT_CLOSE_GRACE_MILLIS);
+ }
+
+ LogReassemblyCallback(Consumer<String> lineConsumer, long closeGraceMillis) {
+ this.lineConsumer = lineConsumer;
+ this.closeGraceMillis = closeGraceMillis;
+ }
+
+ @Override
+ public void onNext(Frame frame) {
+ callbackGate.readLock().lock();
+ try {
+ boolean emit;
+ synchronized (emissionLock) {
+ synchronized (buffers) {
+ if (terminated) {
+ return; // the final drain has run; never append into a buffer that won't be flushed
+ }
+ LogLineBuffer buffer = buffers.computeIfAbsent(frame.getStreamType(), t -> new LogLineBuffer());
+ pendingLines.addAll(buffer.append(frame.getPayload()));
+ if (buffer.hasPendingBytes()) {
+ tailSequences.put(frame.getStreamType(), nextTailSequence++);
+ } else {
+ tailSequences.remove(frame.getStreamType());
+ }
+ }
+ emit = startEmissionIfNeeded();
+ }
+ if (emit) {
+ emitPendingLines();
+ }
+ } finally {
+ callbackGate.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onComplete() {
+ callbackGate.readLock().lock();
+ try {
+ flush();
+ super.onComplete();
+ } finally {
+ callbackGate.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onError(Throwable throwable) {
+ callbackGate.readLock().lock();
+ try {
+ flush();
+ super.onError(throwable);
+ } finally {
+ callbackGate.readLock().unlock();
+ }
+ }
+
+ // Called only once ContainerLifecycleManager has stopped (or confirmed the absence of) the Docker
+ // container. Give its follow-log reader a bounded chance to deliver any queued frames and terminal
+ // callback without blocking lifecycle teardown. If no terminal arrives, close that reader before
+ // flushing so frames cannot continue after the fallback finalization boundary.
+ void closeAfterContainerStopped() {
+ synchronized (emissionLock) {
+ synchronized (buffers) {
+ if (terminated || fallbackFinalization != null) {
+ return;
+ }
+ fallbackFinalization = CLOSE_DRAIN_SCHEDULER.schedule(
+ this::closeTransportAndFlush, closeGraceMillis, TimeUnit.MILLISECONDS);
+ }
+ }
+ }
+
+ // Flush trailing content never terminated by a newline once Docker has signalled the definitive
+ // end of its transport input.
+ private void flush() {
+ boolean emit;
+ synchronized (emissionLock) {
+ synchronized (buffers) {
+ if (terminated) {
+ return;
+ }
+ terminated = true;
+ cancelFallbackFinalization();
+ buffers.entrySet().stream()
+ .sorted(Comparator.comparingLong(entry -> tailSequences.getOrDefault(
+ entry.getKey(), Long.MAX_VALUE)))
+ .forEach(entry -> entry.getValue().flush().ifPresent(pendingLines::addLast));
+ tailSequences.clear();
+ }
+ emit = startEmissionIfNeeded();
+ }
+ if (emit) {
+ emitPendingLines();
+ }
+ }
+
+ private void closeTransportAndFlush() {
+ callbackGate.writeLock().lock();
+ try {
+ try {
+ super.close();
+ } catch (IOException e) {
+ LOG.debugv("Could not close container log transport: {0}", e.getMessage());
+ } finally {
+ flush();
+ }
+ } finally {
+ callbackGate.writeLock().unlock();
+ }
+ }
+
+ private void cancelFallbackFinalization() {
+ if (fallbackFinalization != null) {
+ fallbackFinalization.cancel(false);
+ fallbackFinalization = null;
+ }
+ }
+
+ private boolean startEmissionIfNeeded() {
+ if (emitting) {
+ return false;
+ }
+ synchronized (buffers) {
+ if (pendingLines.isEmpty()) {
+ return false;
+ }
+ }
+ emitting = true;
+ return true;
+ }
+
+ private void emitPendingLines() {
+ while (true) {
+ String line;
+ synchronized (emissionLock) {
+ synchronized (buffers) {
+ line = pendingLines.pollFirst();
+ if (line == null) {
+ emitting = false;
+ return;
+ }
+ }
+ }
+ lineConsumer.accept(line);
+ }
+ }
+ }
+
+ /**
+ * The lifecycle-facing log handle. Closing it is valid only after the owning container has stopped;
+ * it gives Docker a bounded chance to deliver its terminal callback before a non-blocking fallback
+ * closes the reader and finalizes any tail.
+ */
+ static final class ContainerLogHandle implements Closeable {
+
+ private final LogReassemblyCallback callback;
+
+ ContainerLogHandle(LogReassemblyCallback callback) {
+ this.callback = callback;
+ }
+
+ @Override
+ public void close() throws IOException {
+ callback.closeAfterContainerStopped();
+ }
+ }
+
+ /**
+ * Reassembles newline-delimited log lines from Docker multiplexed frame payloads.
+ * A single logical line can be split across several frames (Docker chunks large
+ * writes) and a single frame can carry several lines, so buffering bytes and
+ * emitting exactly one line per {@code '\n'} avoids both splitting long lines and
+ * merging short ones. Bytes are buffered and each complete line decoded as UTF-8,
+ * so a multibyte character straddling a frame boundary is not corrupted.
+ */
+ static final class LogLineBuffer {
+
+ /**
+ * Cap on a single reconstructed line, matching CloudWatch Logs' maximum event size
+ * (256 KB). A longer run with no newline is force-split at the cap, so a container
+ * emitting unbounded output without a newline cannot grow the buffer until the heap
+ * is exhausted. A valid UTF-8 event is split at the limit without cutting a character.
+ */
+ private static final int MAX_LINE_BYTES = 256 * 1024;
+
+ private final ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ /** Continuation bytes still needed to complete the character at the buffer's tail; 0 means the
+ * buffer currently ends on a UTF-8 character boundary. */
+ private int pendingContinuation;
+
+ /** Appends a frame payload and returns the complete lines it produced (possibly none). */
+ List<String> append(byte[] payload) {
+ List<String> lines = new ArrayList<>();
+ for (byte b : payload) {
+ if (b == '\n') {
+ lines.add(takeLine());
+ continue;
+ }
+ // Force-split BEFORE a character that would push the buffer past the cap, so the emitted
+ // event never exceeds MAX_LINE_BYTES and a multibyte character straddling the limit starts
+ // the next event whole instead of inflating this one. Only at a character boundary, and
+ // only for a real character start (not a continuation byte).
+ if (pendingContinuation == 0 && (b & 0xC0) != 0x80
+ && buffer.size() > 0
+ && buffer.size() + utf8CharLength(b) > MAX_LINE_BYTES) {
+ lines.add(takeLine());
+ }
+ buffer.write(b);
+ trackUtf8(b);
+ if ((buffer.size() >= MAX_LINE_BYTES && pendingContinuation == 0)
+ || buffer.size() >= MAX_LINE_BYTES + 3) {
+ // Do not split a valid character whose continuation bytes arrive in a later frame.
+ // Malformed input may never reach a boundary, so allow at most the three continuation
+ // bytes a valid UTF-8 character could need before forcing the buffer to drain.
+ lines.add(takeLine());
+ }
+ }
+ return lines;
+ }
+
+ /** Byte length of the UTF-8 character starting with {@code b} (1 for ASCII or an invalid lead byte). */
+ private static int utf8CharLength(byte b) {
+ int v = b & 0xFF;
+ if ((v & 0x80) == 0x00) return 1; // 0xxxxxxx — ASCII
+ if ((v & 0xE0) == 0xC0) return 2; // 110xxxxx
+ if ((v & 0xF0) == 0xE0) return 3; // 1110xxxx
+ if ((v & 0xF8) == 0xF0) return 4; // 11110xxx
+ return 1; // invalid lead byte
+ }
+
+ /** Returns any buffered content not yet terminated by a newline, clearing the buffer. */
+ Optional<String> flush() {
+ return buffer.size() == 0 ? Optional.empty() : Optional.of(takeLine());
+ }
+
+ boolean hasPendingBytes() {
+ return buffer.size() > 0;
+ }
+
+ /** Updates how many UTF-8 continuation bytes are still expected after appending byte {@code b}. */
+ private void trackUtf8(byte b) {
+ int v = b & 0xFF;
+ if ((v & 0x80) == 0x00) { // 0xxxxxxx — ASCII, self-contained
+ pendingContinuation = 0;
+ } else if ((v & 0xC0) == 0x80) { // 10xxxxxx — continuation byte
+ if (pendingContinuation > 0) {
+ pendingContinuation--;
+ }
+ // a stray continuation (no preceding lead) leaves the tail on a boundary
+ } else if ((v & 0xE0) == 0xC0) { // 110xxxxx — 2-byte lead
+ pendingContinuation = 1;
+ } else if ((v & 0xF0) == 0xE0) { // 1110xxxx — 3-byte lead
+ pendingContinuation = 2;
+ } else if ((v & 0xF8) == 0xF0) { // 11110xxx — 4-byte lead
+ pendingContinuation = 3;
+ } else { // 0xF8-0xFF — invalid lead byte
+ pendingContinuation = 0;
+ }
+ }
+
+ private String takeLine() {
+ String line = buffer.toString(StandardCharsets.UTF_8);
+ buffer.reset();
+ pendingContinuation = 0;
+ return line;
+ }
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
index 1547f6ad1e..fae4632c24 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
@@ -369,11 +369,10 @@ public class CodeBuildRunner implements ContainerTeardown {
}
} finally {
stopFlags.remove(buildId);
- if (logHandle != null) {
- try { logHandle.close(); } catch (Exception ignored) {}
- }
if (containerId != null && runningContainers.remove(buildId, containerId)) {
- lifecycleManager.stopAndRemove(containerId, null);
+ lifecycleManager.stopAndRemove(containerId, logHandle);
+ } else if (logHandle != null) {
+ try { logHandle.close(); } catch (Exception ignored) {}
}
if (workspace != null) {
deleteDirectory(workspace);
diff --git a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java
index c353044ee6..01d7ad4d09 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java
@@ -39,6 +39,8 @@ public class CodeBuildService {
private Map<String, Map<String, SourceCredential>> sourceCredentials = new ConcurrentHashMap<>();
// key: region -> buildId -> build (transient: builds are runtime state)
private final ConcurrentHashMap<String, ConcurrentHashMap<String, Build>> builds = new ConcurrentHashMap<>();
+ // key: region -> buildId -> request buildspec override (transient: builds are runtime state)
+ private final ConcurrentHashMap<String, ConcurrentHashMap<String, String>> buildspecOverrides = new ConcurrentHashMap<>();
// key: region:projectName -> build counter (transient)
private final ConcurrentHashMap<String, AtomicLong> buildCounters = new ConcurrentHashMap<>();
@@ -108,6 +110,10 @@ public class CodeBuildService {
return builds.computeIfAbsent(region, r -> new ConcurrentHashMap<>());
}
+ private Map<String, String> buildspecOverridesFor(String region) {
+ return buildspecOverrides.computeIfAbsent(region, r -> new ConcurrentHashMap<>());
+ }
+
// ---- Projects ----
public Project createProject(String region, String account,
@@ -448,6 +454,9 @@ public class CodeBuildService {
build.setPhases(new CopyOnWriteArrayList<>());
buildsFor(region).put(buildId, build);
+ if (buildspecOverride != null && !buildspecOverride.isBlank()) {
+ buildspecOverridesFor(region).put(buildId, buildspecOverride);
+ }
Build responseBuild = copyBuild(build);
runner.startBuild(region, build, project, buildspecOverride);
@@ -500,8 +509,9 @@ public class CodeBuildService {
public Build retryBuild(String region, String account, String buildId) {
Build original = getBuild(region, buildId);
+ String buildspecOverride = buildspecOverridesFor(region).get(original.getId());
return startBuild(region, account, original.getProjectName(),
- null, original.getEnvironment(), original.getArtifacts(),
+ buildspecOverride, original.getEnvironment(), original.getArtifacts(),
null, original.getTimeoutInMinutes(), null, null);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
index 39b4cab8a2..e7649471e5 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java
@@ -74,7 +74,7 @@ public class EcsService implements ContainerTeardown {
private Map<String, Integer> latestRevisions = new ConcurrentHashMap<>();
// taskArn → EcsTask
private final Map<String, EcsTask> tasks = new ConcurrentHashMap<>();
- // taskArn → EcsTaskHandle (running containers)
+ // taskArn → EcsTaskHandle (running containers or unresolved log readers)
private final Map<String, EcsTaskHandle> taskHandles = new ConcurrentHashMap<>();
// region::clusterName/serviceName → EcsServiceModel
private Map<String, EcsServiceModel> services = new ConcurrentHashMap<>();
@@ -554,7 +554,11 @@ public class EcsService implements ContainerTeardown {
Map<String, Integer> exitCodes = Map.of();
if (dockerMode) {
EcsTaskHandle handle = taskHandles.remove(task.getTaskArn());
- exitCodes = containerManager.stopTaskAndCollectExitCodes(handle);
+ try {
+ exitCodes = containerManager.stopTaskAndCollectExitCodes(handle);
+ } finally {
+ retainUnresolvedLogHandle(task.getTaskArn(), handle);
+ }
}
task.setLastStatus(TaskStatus.STOPPED.name());
@@ -1411,7 +1415,16 @@ public class EcsService implements ContainerTeardown {
private void reconcileTask(String taskArn) {
EcsTask task = tasks.get(taskArn);
- if (task == null || !TaskStatus.RUNNING.name().equals(task.getLastStatus())) {
+ if (task == null) {
+ return;
+ }
+
+ if (TaskStatus.STOPPED.name().equals(task.getLastStatus())) {
+ reconcileStoppedTask(taskArn);
+ return;
+ }
+
+ if (!TaskStatus.RUNNING.name().equals(task.getLastStatus())) {
return;
}
@@ -1463,6 +1476,27 @@ public class EcsService implements ContainerTeardown {
LOG.infov("ECS task {0} reconciled to STOPPED (all containers exited)", taskArn);
}
+ private void reconcileStoppedTask(String taskArn) {
+ EcsTaskHandle claimed = taskHandles.remove(taskArn);
+ if (claimed == null) {
+ return;
+ }
+
+ try {
+ containerManager.stopTaskAndCollectExitCodes(claimed);
+ } finally {
+ // A task can be reported STOPPED even if Docker rejected both teardown attempts.
+ // Keep only the affected log readers and retry their teardown on later reconciliation ticks.
+ retainUnresolvedLogHandle(taskArn, claimed);
+ }
+ }
+
+ private void retainUnresolvedLogHandle(String taskArn, EcsTaskHandle handle) {
+ if (handle != null && handle.hasOpenLogStreams()) {
+ taskHandles.put(taskArn, handle);
+ }
+ }
+
void reconcileServices() {
for (Map.Entry<String, EcsServiceModel> entry : services.entrySet()) {
try {
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
index 6c510a6649..87840200f9 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java
@@ -35,10 +35,12 @@ import org.jboss.logging.Logger;
import java.io.Closeable;
import java.time.Instant;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
/**
* Manages Docker container lifecycle for ECS tasks.
@@ -89,7 +91,7 @@ public class EcsContainerManager {
String taskId = extractTaskId(task.getTaskArn());
Map<String, String> containerIds = new LinkedHashMap<>();
- List<Closeable> logStreams = new ArrayList<>();
+ Map<String, Closeable> logStreamsByContainerId = new LinkedHashMap<>();
List<Container> runtimeContainers = new ArrayList<>();
// Task-level volumes consumed by per-container mountPoints: host volumes map their
@@ -249,7 +251,7 @@ public class EcsContainerManager {
dockerId, logGroup, logStream, region,
"ecs:" + taskDef.getFamily() + ":" + def.getName());
if (logHandle != null) {
- logStreams.add(logHandle);
+ logStreamsByContainerId.put(dockerId, logHandle);
}
}
@@ -258,7 +260,7 @@ public class EcsContainerManager {
task.setDesiredStatus(TaskStatus.RUNNING.name());
task.setStartedAt(Instant.now());
- return new EcsTaskHandle(task.getTaskArn(), containerIds, logStreams);
+ return new EcsTaskHandle(task.getTaskArn(), containerIds, logStreamsByContainerId);
}
/**
@@ -276,15 +278,11 @@ public class EcsContainerManager {
if (handle == null) {
return;
}
- for (Closeable logStream : handle.getLogStreams()) {
- try {
- logStream.close();
- } catch (Exception ignored) {
- }
- }
for (String dockerId : handle.getContainerIds().values()) {
lifecycleManager.stopAndRemove(dockerId, null);
}
+ new ArrayList<>(handle.getLogStreamsByContainerId().keySet())
+ .forEach(dockerId -> finalizeLogStream(handle, dockerId));
}
/**
@@ -298,18 +296,14 @@ public class EcsContainerManager {
return exitCodes;
}
- for (Closeable logStream : handle.getLogStreams()) {
- try {
- logStream.close();
- } catch (Exception ignored) {
- }
- }
-
// Phase 1: stop all containers (no-op for those already exited).
+ Set<String> terminatedContainerIds = new HashSet<>();
for (String dockerId : handle.getContainerIds().values()) {
try {
lifecycleManager.getDockerClient().stopContainerCmd(dockerId).withTimeout(5).exec();
+ terminatedContainerIds.add(dockerId);
} catch (NotFoundException ignored) {
+ terminatedContainerIds.add(dockerId);
} catch (Exception e) {
LOG.warnv("Error stopping ECS container {0}: {1}", dockerId, e.getMessage());
}
@@ -322,14 +316,26 @@ public class EcsContainerManager {
exitCodes.put(name, getExitCodeIfStopped(dockerId));
try {
lifecycleManager.getDockerClient().removeContainerCmd(dockerId).withForce(true).exec();
+ terminatedContainerIds.add(dockerId);
} catch (NotFoundException ignored) {
+ terminatedContainerIds.add(dockerId);
} catch (Exception e) {
LOG.warnv("Error removing ECS container {0}: {1}", dockerId, e.getMessage());
}
}
+ // A force removal terminates Docker's follow-log transport even when the preceding stop failed.
+ // Preserve handles for any container that still may be running after both operations failed.
+ terminatedContainerIds.forEach(dockerId -> finalizeLogStream(handle, dockerId));
return exitCodes;
}
+ private void finalizeLogStream(EcsTaskHandle handle, String dockerId) {
+ Closeable logStream = handle.removeLogStream(dockerId);
+ if (logStream != null) {
+ lifecycleManager.closeLogStreamAfterContainerStop(logStream);
+ }
+ }
+
/**
* Returns the exit code of a container that has already stopped, or {@code null}
* if the container is still running or its state cannot be determined.
diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java
index 9127f3bbbb..3f14cd33e4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java
@@ -1,26 +1,36 @@
package io.github.hectorvent.floci.services.ecs.container;
import java.io.Closeable;
-import java.util.List;
+import java.util.LinkedHashMap;
import java.util.Map;
/**
* Holds the runtime Docker container IDs for a running ECS task.
- * Maps container name → Docker container ID and log stream handle.
+ * Maps container names to Docker IDs and Docker IDs to their log stream handles.
*/
public class EcsTaskHandle {
private final String taskArn;
private final Map<String, String> containerIds; // containerName → dockerId
- private final List<Closeable> logStreams;
+ private final Map<String, Closeable> logStreamsByContainerId;
- public EcsTaskHandle(String taskArn, Map<String, String> containerIds, List<Closeable> logStreams) {
+ public EcsTaskHandle(String taskArn, Map<String, String> containerIds,
+ Map<String, Closeable> logStreamsByContainerId) {
this.taskArn = taskArn;
- this.containerIds = containerIds;
- this.logStreams = logStreams;
+ this.containerIds = new LinkedHashMap<>(containerIds);
+ this.logStreamsByContainerId = new LinkedHashMap<>(logStreamsByContainerId);
}
public String getTaskArn() { return taskArn; }
public Map<String, String> getContainerIds() { return containerIds; }
- public List<Closeable> getLogStreams() { return logStreams; }
+ public Map<String, Closeable> getLogStreamsByContainerId() { return logStreamsByContainerId; }
+
+ /** Removes and returns the log stream that no longer needs task-level ownership. */
+ public Closeable removeLogStream(String containerId) {
+ return logStreamsByContainerId.remove(containerId);
+ }
+
+ public boolean hasOpenLogStreams() {
+ return !logStreamsByContainerId.isEmpty();
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
index df5ce106e0..e7f757d627 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java
@@ -7,6 +7,7 @@ import com.github.dockerjava.api.command.InspectVolumeCmd;
import com.github.dockerjava.api.command.InspectVolumeResponse;
import com.github.dockerjava.api.command.RemoveContainerCmd;
import com.github.dockerjava.api.command.StartContainerCmd;
+import com.github.dockerjava.api.command.StopContainerCmd;
import com.github.dockerjava.api.command.WaitContainerCmd;
import com.github.dockerjava.api.exception.DockerException;
import com.github.dockerjava.api.exception.NotFoundException;
@@ -18,6 +19,8 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Optional;
import java.util.OptionalInt;
@@ -46,6 +49,49 @@ class ContainerLifecycleManagerVolumeTest {
manager = new ContainerLifecycleManager(dockerClient, imageCacheService, containerDetector, portAllocator);
}
+ @Test
+ void stopAndRemoveStopsContainerBeforeClosingItsLogStream() {
+ StopContainerCmd stop = mock(StopContainerCmd.class);
+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class);
+ List<String> operations = new ArrayList<>();
+ doAnswer(invocation -> {
+ operations.add("stop");
+ return null;
+ }).when(stop).exec();
+ doAnswer(invocation -> {
+ operations.add("remove");
+ return null;
+ }).when(remove).exec();
+ when(dockerClient.stopContainerCmd("container-id")).thenReturn(stop);
+ when(stop.withTimeout(5)).thenReturn(stop);
+ when(dockerClient.removeContainerCmd("container-id")).thenReturn(remove);
+ when(remove.withForce(true)).thenReturn(remove);
+
+ manager.stopAndRemove("container-id", () -> operations.add("logs"));
+
+ assertEquals(List.of("stop", "remove", "logs"), operations);
+ }
+
+ @Test
+ void stopAndRemoveForceRemovesAndFinalizesLogStreamWhenContainerStopFails() {
+ StopContainerCmd stop = mock(StopContainerCmd.class);
+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class);
+ List<String> operations = new ArrayList<>();
+ doThrow(new RuntimeException("Docker daemon unavailable")).when(stop).exec();
+ doAnswer(invocation -> {
+ operations.add("remove");
+ return null;
+ }).when(remove).exec();
+ when(dockerClient.stopContainerCmd("container-id")).thenReturn(stop);
+ when(stop.withTimeout(5)).thenReturn(stop);
+ when(dockerClient.removeContainerCmd("container-id")).thenReturn(remove);
+ when(remove.withForce(true)).thenReturn(remove);
+
+ manager.stopAndRemove("container-id", () -> operations.add("logs"));
+
+ assertEquals(List.of("remove", "logs"), operations);
+ }
+
@Test
void volumeExists_returnsTrue_whenVolumeExists() {
InspectVolumeCmd cmd = mock(InspectVolumeCmd.class);
@@ -218,4 +264,3 @@ class ContainerLifecycleManagerVolumeTest {
verify(dockerClient, times(2)).createContainerCmd("busybox:stable");
}
}
-
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java
new file mode 100644
index 0000000000..04a76307cd
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java
@@ -0,0 +1,328 @@
+package io.github.hectorvent.floci.core.common.docker;
+
+import com.github.dockerjava.api.model.Frame;
+import com.github.dockerjava.api.model.StreamType;
+import org.junit.jupiter.api.Test;
+
+import java.io.Closeable;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class ContainerLogStreamerTest {
+
+ private static byte[] utf8(String s) {
+ return s.getBytes(StandardCharsets.UTF_8);
+ }
+
+ @Test
+ void reassemblesSingleLineSplitAcrossFrames() {
+ // A ~1.5 KB stdout line that Docker delivers as multiple frames must arrive
+ // as ONE CloudWatch event, not several (regression for the >1024-byte split).
+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();
+ String longLine = "x".repeat(1500);
+
+ assertEquals(List.of(), buffer.append(utf8(longLine.substring(0, 1024))));
+ assertEquals(List.of(), buffer.append(utf8(longLine.substring(1024))));
+
+ List<String> lines = buffer.append(utf8("\n"));
+ assertEquals(1, lines.size());
+ assertEquals(1500, lines.get(0).length());
+ assertEquals(longLine, lines.get(0));
+ }
+
+ @Test
+ void splitsMultipleLinesPackedIntoOneFrame() {
+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();
+
+ assertEquals(List.of("first", "second", "third"), buffer.append(utf8("first\nsecond\nthird\n")));
+ }
+
+ @Test
+ void flushReturnsTrailingUnterminatedContent() {
+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();
+
+ assertEquals(List.of("complete"), buffer.append(utf8("complete\npartial")));
+ assertEquals("partial", buffer.flush().orElseThrow());
+ assertTrue(buffer.flush().isEmpty(), "buffer should be cleared after flush");
+ }
+
+ @Test
+ void doesNotCorruptMultibyteCharacterSplitAcrossFrames() {
+ // "€" is E2 82 AC in UTF-8; decoding per-frame would corrupt it, decoding the
+ // reassembled line does not.
+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();
+ byte[] euro = utf8("€");
+
+ assertEquals(List.of(), buffer.append(new byte[]{euro[0], euro[1]}));
+ assertEquals(List.of("€"), buffer.append(new byte[]{euro[2], '\n'}));
+ }
+
+ @Test
+ void preservesEmptyLinesInSplitOutput() {
+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();
+
+ // append() splits faithfully; the emit layer is what skips blanks.
+ assertEquals(List.of("a", "", "b"), buffer.append(utf8("a\n\nb\n")));
+ }
+
+ @Test
+ void boundsBufferBySplittingOverlongUnterminatedRuns() {
+ // A 256 KB run with no newline must be force-emitted (bounded), not buffered until the
+ // heap is exhausted — regression guard for the unbounded-growth review finding.
+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();
+ byte[] overlong = new byte[256 * 1024];
+ java.util.Arrays.fill(overlong, (byte) 'a');
+
+ List<String> lines = buffer.append(overlong);
+
+ assertEquals(1, lines.size());
+ assertEquals(256 * 1024, lines.get(0).length());
+ assertTrue(buffer.flush().isEmpty(), "buffer is drained after the forced split");
+ }
+
+ @Test
+ void forcedSplitKeepsEventWithinCapAndDefersStraddlingMultibyteChar() {
+ // Regression for the oversized-event finding: with the buffer one byte below the cap, a 3-byte
+ // '€' would inflate the event to cap+2. The split must instead emit the buffered run WITHIN the
+ // cap and defer the whole '€' to the next event — never cutting the character.
+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();
+ int cap = 256 * 1024;
+ byte[] fill = new byte[cap - 1];
+ java.util.Arrays.fill(fill, (byte) 'a');
+ assertEquals(List.of(), buffer.append(fill));
+
+ List<String> lines = buffer.append(utf8("€"));
+
+ assertEquals(1, lines.size());
+ String event = lines.get(0);
+ assertTrue(event.getBytes(StandardCharsets.UTF_8).length <= cap, "event must not exceed the cap");
+ assertFalse(event.contains("�"), "no half-decoded character in the emitted event");
+ assertTrue(event.chars().allMatch(c -> c == 'a'), "the straddling '€' was deferred, not included");
+ // '€' is buffered whole and emitted intact on the next newline.
+ assertEquals("€", buffer.append(utf8("\n")).get(0));
+ }
+
+ @Test
+ void keepsCharacterIntactWhenItsFinalBytesArriveAtTheCapInAnotherFrame() {
+ // A four-byte character that exactly fills the remaining capacity must not be decoded after its
+ // first frame. The terminal continuation bytes complete the same capped event intact.
+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();
+ int cap = 256 * 1024;
+ byte[] fill = new byte[cap - 4];
+ java.util.Arrays.fill(fill, (byte) 'a');
+ byte[] rocket = utf8("🚀");
+
+ assertEquals(List.of(), buffer.append(fill));
+ assertEquals(List.of(), buffer.append(new byte[]{rocket[0], rocket[1]}));
+ List<String> lines = buffer.append(new byte[]{rocket[2], rocket[3]});
+
+ assertEquals(1, lines.size());
+ assertTrue(lines.get(0).endsWith("🚀"));
+ assertFalse(lines.get(0).contains("�"));
+ }
+
+ @Test
+ void boundsBufferOnRepeatedLeadingBytesThatNeverCompleteACharacter() {
+ // Invalid UTF-8 that never reaches a character boundary (a run of lead bytes) must still be
+ // force-split at the hard ceiling, not buffered without bound.
+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();
+ byte[] leads = new byte[256 * 1024 + 64];
+ java.util.Arrays.fill(leads, (byte) 0xE2); // 1110xxxx lead byte, never followed by continuations
+ List<String> lines = buffer.append(leads);
+ assertFalse(lines.isEmpty(), "a boundary-less lead-byte run must be force-split, not buffered unbounded");
+ }
+
+ @Test
+ void boundsBufferOnStrayContinuationBytes() {
+ // A run of stray continuation bytes (invalid UTF-8, no lead byte) must be force-split at the
+ // cap rather than growing unbounded.
+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();
+ byte[] continuations = new byte[256 * 1024 + 64];
+ java.util.Arrays.fill(continuations, (byte) 0x80); // 10xxxxxx continuation
+ List<String> lines = buffer.append(continuations);
+ assertFalse(lines.isEmpty(), "a continuation-only run must be force-split, not buffered unbounded");
+ }
+
+ @Test
+ void keepsLateUnterminatedFragmentsTogetherUntilTerminal() throws Exception {
+ // A cancelled transport can still deliver an in-flight continuation. It must not flush the first
+ // fragment then let that continuation become a second event. The terminal callback, not elapsed
+ // time, is the boundary that flushes the complete logical line.
+ List<String> emitted = new ArrayList<>();
+ ContainerLogStreamer.LogReassemblyCallback cb =
+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add);
+
+ cb.onNext(new Frame(StreamType.STDOUT, utf8("partial")));
+ cb.close();
+ cb.onNext(new Frame(StreamType.STDOUT, utf8("-late")));
+ assertEquals(List.of(), emitted, "the partial line stays intact until the reader terminates");
+ cb.onComplete();
+
+ assertEquals(List.of("partial-late"), emitted);
+ }
+
+ @Test
+ void lifecycleHandleCloseRetainsDelayedFramesUntilTerminal() throws Exception {
+ // Container shutdown has completed, but the Docker reader can still deliver bytes it already read.
+ // Closing the lifecycle handle must keep the reader attached through its drain period so a delayed
+ // continuation reaches the terminal flush rather than being truncated immediately.
+ List<String> emitted = Collections.synchronizedList(new ArrayList<>());
+ ContainerLogStreamer.LogReassemblyCallback cb =
+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add);
+ Closeable handle = new ContainerLogStreamer.ContainerLogHandle(cb);
+ cb.onNext(new Frame(StreamType.STDOUT, utf8("before")));
+
+ handle.close();
+ cb.onNext(new Frame(StreamType.STDOUT, utf8("-after")));
+ cb.onComplete();
+
+ assertEquals(List.of("before-after"), emitted);
+ }
+
+ @Test
+ void emitsCompletedLinesBeforeTheFinalTailDuringConcurrentLifecycleClose() throws Exception {
+ // The reader begins emitting a complete line, then a lifecycle-confirmed container stop drains a
+ // trailing partial line. The final tail must queue behind the already-complete line, even though
+ // the consumer for that line is still running on a different thread.
+ List<String> emitted = Collections.synchronizedList(new ArrayList<>());
+ CountDownLatch firstEmissionStarted = new CountDownLatch(1);
+ CountDownLatch releaseFirstEmission = new CountDownLatch(1);
+ ContainerLogStreamer.LogReassemblyCallback cb =
+ new ContainerLogStreamer.LogReassemblyCallback(line -> {
+ if ("complete".equals(line)) {
+ firstEmissionStarted.countDown();
+ try {
+ releaseFirstEmission.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(e);
+ }
+ }
+ emitted.add(line);
+ });
+
+ Thread reader = new Thread(() -> cb.onNext(new Frame(StreamType.STDOUT, utf8("complete\npartial"))));
+ reader.start();
+ assertTrue(firstEmissionStarted.await(1, TimeUnit.SECONDS), "the first line should begin emitting");
+
+ Thread closer = new Thread(() -> {
+ cb.closeAfterContainerStopped();
+ });
+ closer.start();
+ Thread terminal = new Thread(cb::onComplete);
+ terminal.start();
+ closer.join();
+ releaseFirstEmission.countDown();
+ reader.join();
+ terminal.join();
+
+ assertEquals(List.of("complete", "partial"), emitted);
+ }
+
+ @Test
+ void framesDeliveredAfterTransportCloseAreStillEmitted() throws Exception {
+ // Cancelling the transport must not permanently stop buffering. A frame the reader delivers after
+ // cancellation is still emitted; the eventual terminal drains the rest.
+ List<String> emitted = new ArrayList<>();
+ ContainerLogStreamer.LogReassemblyCallback cb =
+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add);
+
+ cb.onNext(new Frame(StreamType.STDOUT, utf8("before-grace")));
+ cb.close();
+ cb.onNext(new Frame(StreamType.STDOUT, utf8("-after-close")));
+ cb.onComplete(); // the reader's real terminal drains the reassembled tail
+
+ assertEquals(List.of("before-grace-after-close"), emitted);
+ }
+
+ @Test
+ void onNextAfterTerminalIsIgnored() throws Exception {
+ // A straggler frame arriving after the reader's TERMINAL (the true end of the stream) must not be
+ // appended into a buffer that will never be flushed — the terminated flag, checked under the
+ // lock, drops it deterministically.
+ List<String> emitted = new ArrayList<>();
+ ContainerLogStreamer.LogReassemblyCallback cb =
+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add);
+
+ cb.onNext(new Frame(StreamType.STDOUT, utf8("kept\n")));
+ cb.onComplete(); // true terminal: drains and marks terminated
+ cb.onNext(new Frame(StreamType.STDOUT, utf8("straggler-after-terminal")));
+
+ assertEquals(List.of("kept"), emitted);
+ }
+
+ @Test
+ void reassemblesInFlightFramesDeliveredDuringClose() throws Exception {
+ // Fragments of ONE newline-free write are delivered by the transport reader AFTER close() is
+ // called; the terminal callback then flushes their reassembled line. A separate thread supplies
+ // the reader callbacks, mirroring the transport.
+ List<String> emitted = java.util.Collections.synchronizedList(new ArrayList<>());
+ ContainerLogStreamer.LogReassemblyCallback cb =
+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add);
+
+ cb.onNext(new Frame(StreamType.STDOUT, utf8("frag1")));
+
+ Thread reader = new Thread(() -> {
+ cb.onNext(new Frame(StreamType.STDOUT, utf8("frag2")));
+ cb.onNext(new Frame(StreamType.STDOUT, utf8("frag3\n")));
+ cb.onComplete(); // terminal: drains and unblocks close()
+ });
+ reader.start();
+
+ cb.close();
+ reader.join();
+
+ assertEquals(List.of("frag1frag2frag3"), emitted);
+ }
+
+ @Test
+ void flushesTrailingContentOnComplete() throws Exception {
+ // A clean end (onComplete) flushes any trailing unterminated line.
+ List<String> emitted = new ArrayList<>();
+ ContainerLogStreamer.LogReassemblyCallback cb =
+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add);
+
+ cb.onNext(new Frame(StreamType.STDOUT, utf8("no-newline-tail")));
+ cb.onComplete();
+
+ assertEquals(List.of("no-newline-tail"), emitted);
+ }
+
+ @Test
+ void flushesStreamTailsInTheirLastFrameArrivalOrder() {
+ List<String> emitted = new ArrayList<>();
+ ContainerLogStreamer.LogReassemblyCallback cb =
+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add);
+
+ cb.onNext(new Frame(StreamType.STDERR, utf8("stderr-tail")));
+ cb.onNext(new Frame(StreamType.STDOUT, utf8("stdout-tail")));
+ cb.onComplete();
+
+ assertEquals(List.of("stderr-tail", "stdout-tail"), emitted);
+ }
+
+ @Test
+ void lifecycleHandleFlushesTailWhenDockerNeverSignalsTerminal() throws Exception {
+ List<String> emitted = Collections.synchronizedList(new ArrayList<>());
+ ContainerLogStreamer.LogReassemblyCallback cb =
+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add, 10);
+ Closeable handle = new ContainerLogStreamer.ContainerLogHandle(cb);
+
+ cb.onNext(new Frame(StreamType.STDOUT, utf8("unterminated-tail")));
+ handle.close();
+
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1);
+ while (emitted.isEmpty() && System.nanoTime() < deadline) {
+ Thread.sleep(5);
+ }
+ assertEquals(List.of("unterminated-tail"), emitted);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildServiceTest.java
new file mode 100644
index 0000000000..f869a93bb9
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildServiceTest.java
@@ -0,0 +1,51 @@
+package io.github.hectorvent.floci.services.codebuild;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.services.codebuild.model.Build;
+import io.github.hectorvent.floci.services.codebuild.model.Project;
+import io.github.hectorvent.floci.services.codebuild.model.ProjectArtifacts;
+import io.github.hectorvent.floci.services.codebuild.model.ProjectEnvironment;
+import io.github.hectorvent.floci.services.codebuild.model.ProjectSource;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+class CodeBuildServiceTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT = "000000000000";
+
+ @Test
+ void retryBuildRetainsOriginalBuildspecOverride() {
+ CodeBuildRunner runner = mock(CodeBuildRunner.class);
+ CodeBuildService service = new CodeBuildService(runner, mock(EmulatorConfig.class), null);
+ ProjectSource source = new ProjectSource();
+ source.setType("NO_SOURCE");
+ ProjectArtifacts artifacts = new ProjectArtifacts();
+ artifacts.setType("NO_ARTIFACTS");
+
+ service.createProject(REGION, ACCOUNT, "retry-project", null, source, null, null,
+ artifacts, null, new ProjectEnvironment(), "arn:aws:iam::000000000000:role/codebuild",
+ null, null, null, null, null, null, null);
+
+ String buildspec = "version: 0.2\nphases:\n build:\n commands:\n - echo retry\n";
+ Build original = service.startBuild(REGION, ACCOUNT, "retry-project", buildspec,
+ null, null, null, null, null, null);
+ Build retried = service.retryBuild(REGION, ACCOUNT, original.getId());
+
+ assertNotEquals(original.getId(), retried.getId());
+ ArgumentCaptor<String> buildspecOverrides = ArgumentCaptor.forClass(String.class);
+ verify(runner, times(2)).startBuild(eq(REGION), any(Build.class), any(Project.class),
+ buildspecOverrides.capture());
+ assertEquals(List.of(buildspec, buildspec), buildspecOverrides.getAllValues());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java
index b1d93235f4..bcce5ad76b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java
@@ -12,9 +12,11 @@ import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;
import io.github.hectorvent.floci.services.ecs.model.LaunchType;
import org.junit.jupiter.api.Test;
+import java.io.Closeable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@@ -72,6 +74,50 @@ class EcsServiceTeardownTest {
verify(containerManager, times(1)).stopTask(handle);
}
+ @Test
+ void stoppedTaskRetriesTeardownUntilItsRemainingLogStreamIsReleased() {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().ecs().mock()).thenReturn(false); // docker mode
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+
+ EcsContainerManager containerManager = mock(EcsContainerManager.class);
+ EcsTaskHandle handle = new EcsTaskHandle("task-arn", Map.of("app", "docker-id"),
+ Map.of("docker-id", mock(Closeable.class)));
+ when(containerManager.startTask(any(), any(), any(), anyString())).thenReturn(handle);
+ AtomicInteger teardownAttempts = new AtomicInteger();
+ when(containerManager.stopTaskAndCollectExitCodes(handle)).thenAnswer(ignored -> {
+ if (teardownAttempts.incrementAndGet() == 2) {
+ handle.removeLogStream("docker-id");
+ }
+ return Map.of();
+ });
+
+ EcsService service = new EcsService(
+ new RegionResolver(REGION, "000000000000"),
+ containerManager,
+ config,
+ mock(EcsLoadBalancerRegistrar.class),
+ new SingleUseStorageFactory());
+ service.initializeStorage();
+
+ ContainerDefinition cd = new ContainerDefinition();
+ cd.setName("app");
+ cd.setImage("nginx:alpine");
+ service.registerTaskDefinition("retry-fam", List.of(cd), null, null, null,
+ null, null, List.of(), REGION);
+ String taskArn = service.runTask(null, "retry-fam", 1, LaunchType.FARGATE, null, null,
+ List.of(), null, REGION).getFirst().getTaskArn();
+
+ service.stopTask(null, taskArn, null, REGION);
+ verify(containerManager, times(1)).stopTaskAndCollectExitCodes(handle);
+
+ service.reconcile();
+ verify(containerManager, times(2)).stopTaskAndCollectExitCodes(handle);
+
+ service.reconcile();
+ verify(containerManager, times(2)).stopTaskAndCollectExitCodes(handle);
+ }
+
private static final class SingleUseStorageFactory extends StorageFactory {
private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java
new file mode 100644
index 0000000000..928c0f0b21
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java
@@ -0,0 +1,121 @@
+package io.github.hectorvent.floci.services.ecs.container;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.RemoveContainerCmd;
+import com.github.dockerjava.api.command.StopContainerCmd;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
+import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;
+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
+import io.github.hectorvent.floci.services.ssm.SsmService;
+import org.junit.jupiter.api.Test;
+
+import java.io.Closeable;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class EcsContainerManagerLifecycleTest {
+
+ @Test
+ void finalizesTaskLogStreamsAfterForceRemovingAContainerWhoseStopFails() {
+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ DockerClient dockerClient = mock(DockerClient.class);
+ StopContainerCmd stop = mock(StopContainerCmd.class);
+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class);
+ Closeable logStream = mock(Closeable.class);
+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+ when(dockerClient.stopContainerCmd("docker-id")).thenReturn(stop);
+ when(stop.withTimeout(5)).thenReturn(stop);
+ when(dockerClient.removeContainerCmd("docker-id")).thenReturn(remove);
+ when(remove.withForce(true)).thenReturn(remove);
+ doThrow(new RuntimeException("Docker daemon unavailable")).when(stop).exec();
+
+ EcsContainerManager manager = manager(lifecycleManager);
+ EcsTaskHandle handle = new EcsTaskHandle("task-arn", Map.of("app", "docker-id"),
+ Map.of("docker-id", logStream));
+
+ manager.stopTaskAndCollectExitCodes(handle);
+
+ verify(remove).exec();
+ verify(lifecycleManager).closeLogStreamAfterContainerStop(logStream);
+ }
+
+ @Test
+ void finalizesTaskLogStreamsAfterEveryContainerStops() {
+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ DockerClient dockerClient = mock(DockerClient.class);
+ StopContainerCmd stop = mock(StopContainerCmd.class);
+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class);
+ Closeable logStream = mock(Closeable.class);
+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+ when(dockerClient.stopContainerCmd("docker-id")).thenReturn(stop);
+ when(stop.withTimeout(5)).thenReturn(stop);
+ when(dockerClient.removeContainerCmd("docker-id")).thenReturn(remove);
+ when(remove.withForce(true)).thenReturn(remove);
+
+ EcsContainerManager manager = manager(lifecycleManager);
+ EcsTaskHandle handle = new EcsTaskHandle("task-arn", Map.of("app", "docker-id"),
+ Map.of("docker-id", logStream));
+
+ manager.stopTaskAndCollectExitCodes(handle);
+
+ verify(lifecycleManager).closeLogStreamAfterContainerStop(logStream);
+ }
+
+ @Test
+ void preservesOnlyTheLogStreamForAContainerThatCouldNotStopOrBeRemoved() {
+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ DockerClient dockerClient = mock(DockerClient.class);
+ StopContainerCmd failedStop = mock(StopContainerCmd.class);
+ StopContainerCmd successfulStop = mock(StopContainerCmd.class);
+ RemoveContainerCmd failedRemove = mock(RemoveContainerCmd.class);
+ RemoveContainerCmd successfulRemove = mock(RemoveContainerCmd.class);
+ Closeable retainedLogStream = mock(Closeable.class);
+ Closeable finalizedLogStream = mock(Closeable.class);
+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+ when(dockerClient.stopContainerCmd("running-id")).thenReturn(failedStop);
+ when(failedStop.withTimeout(5)).thenReturn(failedStop);
+ doThrow(new RuntimeException("stop failed")).when(failedStop).exec();
+ when(dockerClient.stopContainerCmd("stopped-id")).thenReturn(successfulStop);
+ when(successfulStop.withTimeout(5)).thenReturn(successfulStop);
+ when(dockerClient.removeContainerCmd("running-id")).thenReturn(failedRemove);
+ when(failedRemove.withForce(true)).thenReturn(failedRemove);
+ doThrow(new RuntimeException("remove failed")).when(failedRemove).exec();
+ when(dockerClient.removeContainerCmd("stopped-id")).thenReturn(successfulRemove);
+ when(successfulRemove.withForce(true)).thenReturn(successfulRemove);
+
+ Map<String, String> containerIds = new LinkedHashMap<>();
+ containerIds.put("running", "running-id");
+ containerIds.put("stopped", "stopped-id");
+ EcsTaskHandle handle = new EcsTaskHandle("task-arn", containerIds,
+ Map.of("running-id", retainedLogStream, "stopped-id", finalizedLogStream));
+
+ manager(lifecycleManager).stopTaskAndCollectExitCodes(handle);
+
+ verify(lifecycleManager, never()).closeLogStreamAfterContainerStop(retainedLogStream);
+ verify(lifecycleManager).closeLogStreamAfterContainerStop(finalizedLogStream);
+ assertTrue(handle.hasOpenLogStreams());
+ assertSame(retainedLogStream, handle.getLogStreamsByContainerId().get("running-id"));
+ assertFalse(handle.getLogStreamsByContainerId().containsKey("stopped-id"));
+ }
+
+ private static EcsContainerManager manager(ContainerLifecycleManager lifecycleManager) {
+ return new EcsContainerManager(
+ mock(ContainerBuilder.class), lifecycleManager, mock(ContainerLogStreamer.class),
+ mock(ContainerDetector.class), mock(EmulatorConfig.class), mock(RegionResolver.class),
+ mock(LaunchedContainerAwsEnv.class), mock(SsmService.class), mock(SecretsManagerService.class));
+ }
+}