Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): reassemble container log lines split across Docker frames (#1953)
Aniket Shukla6344852540ce
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javaindex 65844ff352..42a35c8a89 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -164,34 +164,46 @@ public class ContainerLifecycleManager {public void stopAndRemove(String containerId, Closeable logStream) {LOG.infov("Stopping container {0}", containerId);- // Close log stream first- if (logStream != null) {- try {- logStream.close();- } catch (Exception e) {- LOG.debugv("Error closing log stream: {0}", e.getMessage());- }- }-- // Stop container+ boolean stoppedOrMissing = false;try {dockerClient.stopContainerCmd(containerId).withTimeout(5).exec();+ stoppedOrMissing = true;} catch (NotFoundException e) {LOG.debugv("Container {0} not found (already removed)", containerId);- return;+ stoppedOrMissing = true;} catch (Exception e) {LOG.warnv("Error stopping container {0}: {1}", containerId, e.getMessage());}- // Remove container+ // Force removal is the recovery path when Docker could not stop the container cleanly. It also+ // terminates the follow-log transport, allowing its terminal callback to drain the final tail.+ boolean removedOrMissing = false;try {dockerClient.removeContainerCmd(containerId).withForce(true).exec();+ removedOrMissing = true;LOG.debugv("Removed container {0}", containerId);} catch (NotFoundException e) {// Already gone+ removedOrMissing = true;} catch (Exception e) {LOG.warnv("Error removing container {0}: {1}", containerId, e.getMessage());}++ if (logStream != null && (stoppedOrMissing || removedOrMissing)) {+ closeLogStreamAfterContainerStop(logStream);+ }+ }++ /**+ * Releases lifecycle ownership of a log stream after its container has stopped. Docker's terminal+ * callback normally flushes the final tail; a bounded fallback handles a broken transport.+ */+ public void closeLogStreamAfterContainerStop(Closeable logStream) {+ try {+ logStream.close();+ } catch (Exception e) {+ LOG.debugv("Error closing log stream: {0}", e.getMessage());+ }}/**diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.javaindex 0996536ff3..0fca8b72a6 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java@@ -5,17 +5,32 @@ import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsServiceimport com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.async.ResultCallback;import com.github.dockerjava.api.model.Frame;+import com.github.dockerjava.api.model.StreamType;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.io.ByteArrayOutputStream;import java.io.Closeable;+import java.io.IOException;import java.nio.charset.StandardCharsets;import java.time.LocalDate;import java.time.format.DateTimeFormatter;+import java.util.ArrayDeque;+import java.util.ArrayList;+import java.util.Comparator;+import java.util.Deque;+import java.util.EnumMap;import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.Optional;+import java.util.concurrent.Executors;+import java.util.concurrent.ScheduledExecutorService;+import java.util.concurrent.ScheduledFuture;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.locks.ReentrantReadWriteLock;+import java.util.function.Consumer;/*** Streams Docker container logs to both the Floci console logger and CloudWatch Logs.@@ -38,26 +53,39 @@ public class ContainerLogStreamer {/*** Attaches a log stream to a container and forwards logs to CloudWatch Logs.- * Returns a Closeable handle that should be closed when the container is stopped.+ * Returns a lifecycle handle that should be closed after the container stops. Closing the handle+ * allows Docker's reader a bounded drain period before a non-blocking fallback flushes the tail.** @param containerId Docker container ID* @param logGroup CloudWatch log group name (e.g., "/aws/lambda/myFunction")* @param logStream CloudWatch log stream name (e.g., "2024/01/15/[$LATEST]abc123")* @param region AWS region for CloudWatch Logs* @param logPrefix prefix for console logging (e.g., "lambda:myFunction")- * @return Closeable handle to stop the log stream+ * @return Closeable lifecycle handle for the log stream*/public Closeable attach(String containerId, String logGroup, String logStream,String region, String logPrefix) {ensureLogGroupAndStream(logGroup, logStream, region);+ // Trim trailing whitespace and drop blank lines, then fan each reassembled line out to the+ // console logger and CloudWatch Logs.+ Consumer<String> emitter = line -> {+ String trimmed = line.stripTrailing();+ if (!trimmed.isEmpty()) {+ LOG.infov("[{0}] {1}", logPrefix, trimmed);+ forwardToCloudWatchLogs(logGroup, logStream, region, trimmed);+ }+ };+try {- return dockerClient.logContainerCmd(containerId)+ LogReassemblyCallback callback = new LogReassemblyCallback(emitter);+ dockerClient.logContainerCmd(containerId).withStdOut(true).withStdErr(true).withFollowStream(true).withTimestamps(false)- .exec(frameCallback(logGroup, logStream, region, logPrefix));+ .exec(callback);+ return new ContainerLogHandle(callback);} catch (Exception e) {LOG.warnv("Could not attach log stream for container {0}: {1}", containerId, e.getMessage());return null;@@ -149,4 +177,312 @@ public class ContainerLogStreamer {LOG.debugv("Could not forward log line to CloudWatch Logs: {0}", e.getMessage());}}++ /**+ * Docker frame callback that reassembles multiplexed stdout/stderr frames into newline-delimited+ * lines and forwards each to {@code lineConsumer}. One logical line can span several frames and one+ * frame can carry several lines, so bytes are buffered per stream and emitted one line at a time+ * (matching real AWS Lambda).+ */+ static final class LogReassemblyCallback extends ResultCallback.Adapter<Frame> {++ private static final ScheduledExecutorService CLOSE_DRAIN_SCHEDULER =+ Executors.newSingleThreadScheduledExecutor(runnable -> {+ Thread thread = new Thread(runnable, "floci-container-log-close");+ thread.setDaemon(true);+ return thread;+ });+ private static final long DEFAULT_CLOSE_GRACE_MILLIS = 2_000;++ private final Map<StreamType, LogLineBuffer> buffers = new EnumMap<>(StreamType.class);+ private final Map<StreamType, Long> tailSequences = new EnumMap<>(StreamType.class);+ private final Deque<String> pendingLines = new ArrayDeque<>();+ private final Object emissionLock = new Object();+ // A fair write lock makes fallback finalization wait for callbacks already in progress, then blocks+ // any later callback until the transport has been closed and the final tail has been flushed.+ private final ReentrantReadWriteLock callbackGate = new ReentrantReadWriteLock(true);+ private final Consumer<String> lineConsumer;+ private final long closeGraceMillis;+ // Guarded by 'buffers'. Set only by the Docker transport terminal callback; afterwards onNext+ // refuses to append, so nothing is ever buffered past the final flush.+ private boolean terminated;+ // Guarded by 'buffers'. The most recent callback with an unfinished tail gets the largest value.+ private long nextTailSequence;+ // Guarded by 'buffers'. Cancels when Docker supplies its terminal callback before the fallback.+ private ScheduledFuture<?> fallbackFinalization;+ // Guarded by 'emissionLock'. Exactly one caller drains pendingLines at a time, preserving Docker+ // callback order even when a terminal callback and onNext run on different threads.+ private boolean emitting;++ LogReassemblyCallback(Consumer<String> lineConsumer) {+ this(lineConsumer, DEFAULT_CLOSE_GRACE_MILLIS);+ }++ LogReassemblyCallback(Consumer<String> lineConsumer, long closeGraceMillis) {+ this.lineConsumer = lineConsumer;+ this.closeGraceMillis = closeGraceMillis;+ }++ @Override+ public void onNext(Frame frame) {+ callbackGate.readLock().lock();+ try {+ boolean emit;+ synchronized (emissionLock) {+ synchronized (buffers) {+ if (terminated) {+ return; // the final drain has run; never append into a buffer that won't be flushed+ }+ LogLineBuffer buffer = buffers.computeIfAbsent(frame.getStreamType(), t -> new LogLineBuffer());+ pendingLines.addAll(buffer.append(frame.getPayload()));+ if (buffer.hasPendingBytes()) {+ tailSequences.put(frame.getStreamType(), nextTailSequence++);+ } else {+ tailSequences.remove(frame.getStreamType());+ }+ }+ emit = startEmissionIfNeeded();+ }+ if (emit) {+ emitPendingLines();+ }+ } finally {+ callbackGate.readLock().unlock();+ }+ }++ @Override+ public void onComplete() {+ callbackGate.readLock().lock();+ try {+ flush();+ super.onComplete();+ } finally {+ callbackGate.readLock().unlock();+ }+ }++ @Override+ public void onError(Throwable throwable) {+ callbackGate.readLock().lock();+ try {+ flush();+ super.onError(throwable);+ } finally {+ callbackGate.readLock().unlock();+ }+ }++ // Called only once ContainerLifecycleManager has stopped (or confirmed the absence of) the Docker+ // container. Give its follow-log reader a bounded chance to deliver any queued frames and terminal+ // callback without blocking lifecycle teardown. If no terminal arrives, close that reader before+ // flushing so frames cannot continue after the fallback finalization boundary.+ void closeAfterContainerStopped() {+ synchronized (emissionLock) {+ synchronized (buffers) {+ if (terminated || fallbackFinalization != null) {+ return;+ }+ fallbackFinalization = CLOSE_DRAIN_SCHEDULER.schedule(+ this::closeTransportAndFlush, closeGraceMillis, TimeUnit.MILLISECONDS);+ }+ }+ }++ // Flush trailing content never terminated by a newline once Docker has signalled the definitive+ // end of its transport input.+ private void flush() {+ boolean emit;+ synchronized (emissionLock) {+ synchronized (buffers) {+ if (terminated) {+ return;+ }+ terminated = true;+ cancelFallbackFinalization();+ buffers.entrySet().stream()+ .sorted(Comparator.comparingLong(entry -> tailSequences.getOrDefault(+ entry.getKey(), Long.MAX_VALUE)))+ .forEach(entry -> entry.getValue().flush().ifPresent(pendingLines::addLast));+ tailSequences.clear();+ }+ emit = startEmissionIfNeeded();+ }+ if (emit) {+ emitPendingLines();+ }+ }++ private void closeTransportAndFlush() {+ callbackGate.writeLock().lock();+ try {+ try {+ super.close();+ } catch (IOException e) {+ LOG.debugv("Could not close container log transport: {0}", e.getMessage());+ } finally {+ flush();+ }+ } finally {+ callbackGate.writeLock().unlock();+ }+ }++ private void cancelFallbackFinalization() {+ if (fallbackFinalization != null) {+ fallbackFinalization.cancel(false);+ fallbackFinalization = null;+ }+ }++ private boolean startEmissionIfNeeded() {+ if (emitting) {+ return false;+ }+ synchronized (buffers) {+ if (pendingLines.isEmpty()) {+ return false;+ }+ }+ emitting = true;+ return true;+ }++ private void emitPendingLines() {+ while (true) {+ String line;+ synchronized (emissionLock) {+ synchronized (buffers) {+ line = pendingLines.pollFirst();+ if (line == null) {+ emitting = false;+ return;+ }+ }+ }+ lineConsumer.accept(line);+ }+ }+ }++ /**+ * The lifecycle-facing log handle. Closing it is valid only after the owning container has stopped;+ * it gives Docker a bounded chance to deliver its terminal callback before a non-blocking fallback+ * closes the reader and finalizes any tail.+ */+ static final class ContainerLogHandle implements Closeable {++ private final LogReassemblyCallback callback;++ ContainerLogHandle(LogReassemblyCallback callback) {+ this.callback = callback;+ }++ @Override+ public void close() throws IOException {+ callback.closeAfterContainerStopped();+ }+ }++ /**+ * Reassembles newline-delimited log lines from Docker multiplexed frame payloads.+ * A single logical line can be split across several frames (Docker chunks large+ * writes) and a single frame can carry several lines, so buffering bytes and+ * emitting exactly one line per {@code '\n'} avoids both splitting long lines and+ * merging short ones. Bytes are buffered and each complete line decoded as UTF-8,+ * so a multibyte character straddling a frame boundary is not corrupted.+ */+ static final class LogLineBuffer {++ /**+ * Cap on a single reconstructed line, matching CloudWatch Logs' maximum event size+ * (256 KB). A longer run with no newline is force-split at the cap, so a container+ * emitting unbounded output without a newline cannot grow the buffer until the heap+ * is exhausted. A valid UTF-8 event is split at the limit without cutting a character.+ */+ private static final int MAX_LINE_BYTES = 256 * 1024;++ private final ByteArrayOutputStream buffer = new ByteArrayOutputStream();++ /** Continuation bytes still needed to complete the character at the buffer's tail; 0 means the+ * buffer currently ends on a UTF-8 character boundary. */+ private int pendingContinuation;++ /** Appends a frame payload and returns the complete lines it produced (possibly none). */+ List<String> append(byte[] payload) {+ List<String> lines = new ArrayList<>();+ for (byte b : payload) {+ if (b == '\n') {+ lines.add(takeLine());+ continue;+ }+ // Force-split BEFORE a character that would push the buffer past the cap, so the emitted+ // event never exceeds MAX_LINE_BYTES and a multibyte character straddling the limit starts+ // the next event whole instead of inflating this one. Only at a character boundary, and+ // only for a real character start (not a continuation byte).+ if (pendingContinuation == 0 && (b & 0xC0) != 0x80+ && buffer.size() > 0+ && buffer.size() + utf8CharLength(b) > MAX_LINE_BYTES) {+ lines.add(takeLine());+ }+ buffer.write(b);+ trackUtf8(b);+ if ((buffer.size() >= MAX_LINE_BYTES && pendingContinuation == 0)+ || buffer.size() >= MAX_LINE_BYTES + 3) {+ // Do not split a valid character whose continuation bytes arrive in a later frame.+ // Malformed input may never reach a boundary, so allow at most the three continuation+ // bytes a valid UTF-8 character could need before forcing the buffer to drain.+ lines.add(takeLine());+ }+ }+ return lines;+ }++ /** Byte length of the UTF-8 character starting with {@code b} (1 for ASCII or an invalid lead byte). */+ private static int utf8CharLength(byte b) {+ int v = b & 0xFF;+ if ((v & 0x80) == 0x00) return 1; // 0xxxxxxx — ASCII+ if ((v & 0xE0) == 0xC0) return 2; // 110xxxxx+ if ((v & 0xF0) == 0xE0) return 3; // 1110xxxx+ if ((v & 0xF8) == 0xF0) return 4; // 11110xxx+ return 1; // invalid lead byte+ }++ /** Returns any buffered content not yet terminated by a newline, clearing the buffer. */+ Optional<String> flush() {+ return buffer.size() == 0 ? Optional.empty() : Optional.of(takeLine());+ }++ boolean hasPendingBytes() {+ return buffer.size() > 0;+ }++ /** Updates how many UTF-8 continuation bytes are still expected after appending byte {@code b}. */+ private void trackUtf8(byte b) {+ int v = b & 0xFF;+ if ((v & 0x80) == 0x00) { // 0xxxxxxx — ASCII, self-contained+ pendingContinuation = 0;+ } else if ((v & 0xC0) == 0x80) { // 10xxxxxx — continuation byte+ if (pendingContinuation > 0) {+ pendingContinuation--;+ }+ // a stray continuation (no preceding lead) leaves the tail on a boundary+ } else if ((v & 0xE0) == 0xC0) { // 110xxxxx — 2-byte lead+ pendingContinuation = 1;+ } else if ((v & 0xF0) == 0xE0) { // 1110xxxx — 3-byte lead+ pendingContinuation = 2;+ } else if ((v & 0xF8) == 0xF0) { // 11110xxx — 4-byte lead+ pendingContinuation = 3;+ } else { // 0xF8-0xFF — invalid lead byte+ pendingContinuation = 0;+ }+ }++ private String takeLine() {+ String line = buffer.toString(StandardCharsets.UTF_8);+ buffer.reset();+ pendingContinuation = 0;+ return line;+ }+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.javaindex 1547f6ad1e..fae4632c24 100644--- a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java+++ b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java@@ -369,11 +369,10 @@ public class CodeBuildRunner implements ContainerTeardown {}} finally {stopFlags.remove(buildId);- if (logHandle != null) {- try { logHandle.close(); } catch (Exception ignored) {}- }if (containerId != null && runningContainers.remove(buildId, containerId)) {- lifecycleManager.stopAndRemove(containerId, null);+ lifecycleManager.stopAndRemove(containerId, logHandle);+ } else if (logHandle != null) {+ try { logHandle.close(); } catch (Exception ignored) {}}if (workspace != null) {deleteDirectory(workspace);diff --git a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.javaindex c353044ee6..01d7ad4d09 100644--- a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java+++ b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java@@ -39,6 +39,8 @@ public class CodeBuildService {private Map<String, Map<String, SourceCredential>> sourceCredentials = new ConcurrentHashMap<>();// key: region -> buildId -> build (transient: builds are runtime state)private final ConcurrentHashMap<String, ConcurrentHashMap<String, Build>> builds = new ConcurrentHashMap<>();+ // key: region -> buildId -> request buildspec override (transient: builds are runtime state)+ private final ConcurrentHashMap<String, ConcurrentHashMap<String, String>> buildspecOverrides = new ConcurrentHashMap<>();// key: region:projectName -> build counter (transient)private final ConcurrentHashMap<String, AtomicLong> buildCounters = new ConcurrentHashMap<>();@@ -108,6 +110,10 @@ public class CodeBuildService {return builds.computeIfAbsent(region, r -> new ConcurrentHashMap<>());}+ private Map<String, String> buildspecOverridesFor(String region) {+ return buildspecOverrides.computeIfAbsent(region, r -> new ConcurrentHashMap<>());+ }+// ---- Projects ----public Project createProject(String region, String account,@@ -448,6 +454,9 @@ public class CodeBuildService {build.setPhases(new CopyOnWriteArrayList<>());buildsFor(region).put(buildId, build);+ if (buildspecOverride != null && !buildspecOverride.isBlank()) {+ buildspecOverridesFor(region).put(buildId, buildspecOverride);+ }Build responseBuild = copyBuild(build);runner.startBuild(region, build, project, buildspecOverride);@@ -500,8 +509,9 @@ public class CodeBuildService {public Build retryBuild(String region, String account, String buildId) {Build original = getBuild(region, buildId);+ String buildspecOverride = buildspecOverridesFor(region).get(original.getId());return startBuild(region, account, original.getProjectName(),- null, original.getEnvironment(), original.getArtifacts(),+ buildspecOverride, original.getEnvironment(), original.getArtifacts(),null, original.getTimeoutInMinutes(), null, null);}diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.javaindex 39b4cab8a2..e7649471e5 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/EcsService.java@@ -74,7 +74,7 @@ public class EcsService implements ContainerTeardown {private Map<String, Integer> latestRevisions = new ConcurrentHashMap<>();// taskArn → EcsTaskprivate final Map<String, EcsTask> tasks = new ConcurrentHashMap<>();- // taskArn → EcsTaskHandle (running containers)+ // taskArn → EcsTaskHandle (running containers or unresolved log readers)private final Map<String, EcsTaskHandle> taskHandles = new ConcurrentHashMap<>();// region::clusterName/serviceName → EcsServiceModelprivate Map<String, EcsServiceModel> services = new ConcurrentHashMap<>();@@ -554,7 +554,11 @@ public class EcsService implements ContainerTeardown {Map<String, Integer> exitCodes = Map.of();if (dockerMode) {EcsTaskHandle handle = taskHandles.remove(task.getTaskArn());- exitCodes = containerManager.stopTaskAndCollectExitCodes(handle);+ try {+ exitCodes = containerManager.stopTaskAndCollectExitCodes(handle);+ } finally {+ retainUnresolvedLogHandle(task.getTaskArn(), handle);+ }}task.setLastStatus(TaskStatus.STOPPED.name());@@ -1411,7 +1415,16 @@ public class EcsService implements ContainerTeardown {private void reconcileTask(String taskArn) {EcsTask task = tasks.get(taskArn);- if (task == null || !TaskStatus.RUNNING.name().equals(task.getLastStatus())) {+ if (task == null) {+ return;+ }++ if (TaskStatus.STOPPED.name().equals(task.getLastStatus())) {+ reconcileStoppedTask(taskArn);+ return;+ }++ if (!TaskStatus.RUNNING.name().equals(task.getLastStatus())) {return;}@@ -1463,6 +1476,27 @@ public class EcsService implements ContainerTeardown {LOG.infov("ECS task {0} reconciled to STOPPED (all containers exited)", taskArn);}+ private void reconcileStoppedTask(String taskArn) {+ EcsTaskHandle claimed = taskHandles.remove(taskArn);+ if (claimed == null) {+ return;+ }++ try {+ containerManager.stopTaskAndCollectExitCodes(claimed);+ } finally {+ // A task can be reported STOPPED even if Docker rejected both teardown attempts.+ // Keep only the affected log readers and retry their teardown on later reconciliation ticks.+ retainUnresolvedLogHandle(taskArn, claimed);+ }+ }++ private void retainUnresolvedLogHandle(String taskArn, EcsTaskHandle handle) {+ if (handle != null && handle.hasOpenLogStreams()) {+ taskHandles.put(taskArn, handle);+ }+ }+void reconcileServices() {for (Map.Entry<String, EcsServiceModel> entry : services.entrySet()) {try {diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.javaindex 6c510a6649..87840200f9 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java@@ -35,10 +35,12 @@ import org.jboss.logging.Logger;import java.io.Closeable;import java.time.Instant;import java.util.ArrayList;+import java.util.HashSet;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.Optional;+import java.util.Set;/*** Manages Docker container lifecycle for ECS tasks.@@ -89,7 +91,7 @@ public class EcsContainerManager {String taskId = extractTaskId(task.getTaskArn());Map<String, String> containerIds = new LinkedHashMap<>();- List<Closeable> logStreams = new ArrayList<>();+ Map<String, Closeable> logStreamsByContainerId = new LinkedHashMap<>();List<Container> runtimeContainers = new ArrayList<>();// Task-level volumes consumed by per-container mountPoints: host volumes map their@@ -249,7 +251,7 @@ public class EcsContainerManager {dockerId, logGroup, logStream, region,"ecs:" + taskDef.getFamily() + ":" + def.getName());if (logHandle != null) {- logStreams.add(logHandle);+ logStreamsByContainerId.put(dockerId, logHandle);}}@@ -258,7 +260,7 @@ public class EcsContainerManager {task.setDesiredStatus(TaskStatus.RUNNING.name());task.setStartedAt(Instant.now());- return new EcsTaskHandle(task.getTaskArn(), containerIds, logStreams);+ return new EcsTaskHandle(task.getTaskArn(), containerIds, logStreamsByContainerId);}/**@@ -276,15 +278,11 @@ public class EcsContainerManager {if (handle == null) {return;}- for (Closeable logStream : handle.getLogStreams()) {- try {- logStream.close();- } catch (Exception ignored) {- }- }for (String dockerId : handle.getContainerIds().values()) {lifecycleManager.stopAndRemove(dockerId, null);}+ new ArrayList<>(handle.getLogStreamsByContainerId().keySet())+ .forEach(dockerId -> finalizeLogStream(handle, dockerId));}/**@@ -298,18 +296,14 @@ public class EcsContainerManager {return exitCodes;}- for (Closeable logStream : handle.getLogStreams()) {- try {- logStream.close();- } catch (Exception ignored) {- }- }-// Phase 1: stop all containers (no-op for those already exited).+ Set<String> terminatedContainerIds = new HashSet<>();for (String dockerId : handle.getContainerIds().values()) {try {lifecycleManager.getDockerClient().stopContainerCmd(dockerId).withTimeout(5).exec();+ terminatedContainerIds.add(dockerId);} catch (NotFoundException ignored) {+ terminatedContainerIds.add(dockerId);} catch (Exception e) {LOG.warnv("Error stopping ECS container {0}: {1}", dockerId, e.getMessage());}@@ -322,14 +316,26 @@ public class EcsContainerManager {exitCodes.put(name, getExitCodeIfStopped(dockerId));try {lifecycleManager.getDockerClient().removeContainerCmd(dockerId).withForce(true).exec();+ terminatedContainerIds.add(dockerId);} catch (NotFoundException ignored) {+ terminatedContainerIds.add(dockerId);} catch (Exception e) {LOG.warnv("Error removing ECS container {0}: {1}", dockerId, e.getMessage());}}+ // A force removal terminates Docker's follow-log transport even when the preceding stop failed.+ // Preserve handles for any container that still may be running after both operations failed.+ terminatedContainerIds.forEach(dockerId -> finalizeLogStream(handle, dockerId));return exitCodes;}+ private void finalizeLogStream(EcsTaskHandle handle, String dockerId) {+ Closeable logStream = handle.removeLogStream(dockerId);+ if (logStream != null) {+ lifecycleManager.closeLogStreamAfterContainerStop(logStream);+ }+ }+/*** Returns the exit code of a container that has already stopped, or {@code null}* if the container is still running or its state cannot be determined.diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.javaindex 9127f3bbbb..3f14cd33e4 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsTaskHandle.java@@ -1,26 +1,36 @@package io.github.hectorvent.floci.services.ecs.container;import java.io.Closeable;-import java.util.List;+import java.util.LinkedHashMap;import java.util.Map;/*** Holds the runtime Docker container IDs for a running ECS task.- * Maps container name → Docker container ID and log stream handle.+ * Maps container names to Docker IDs and Docker IDs to their log stream handles.*/public class EcsTaskHandle {private final String taskArn;private final Map<String, String> containerIds; // containerName → dockerId- private final List<Closeable> logStreams;+ private final Map<String, Closeable> logStreamsByContainerId;- public EcsTaskHandle(String taskArn, Map<String, String> containerIds, List<Closeable> logStreams) {+ public EcsTaskHandle(String taskArn, Map<String, String> containerIds,+ Map<String, Closeable> logStreamsByContainerId) {this.taskArn = taskArn;- this.containerIds = containerIds;- this.logStreams = logStreams;+ this.containerIds = new LinkedHashMap<>(containerIds);+ this.logStreamsByContainerId = new LinkedHashMap<>(logStreamsByContainerId);}public String getTaskArn() { return taskArn; }public Map<String, String> getContainerIds() { return containerIds; }- public List<Closeable> getLogStreams() { return logStreams; }+ public Map<String, Closeable> getLogStreamsByContainerId() { return logStreamsByContainerId; }++ /** Removes and returns the log stream that no longer needs task-level ownership. */+ public Closeable removeLogStream(String containerId) {+ return logStreamsByContainerId.remove(containerId);+ }++ public boolean hasOpenLogStreams() {+ return !logStreamsByContainerId.isEmpty();+ }}diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.javaindex df5ce106e0..e7f757d627 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerVolumeTest.java@@ -7,6 +7,7 @@ import com.github.dockerjava.api.command.InspectVolumeCmd;import com.github.dockerjava.api.command.InspectVolumeResponse;import com.github.dockerjava.api.command.RemoveContainerCmd;import com.github.dockerjava.api.command.StartContainerCmd;+import com.github.dockerjava.api.command.StopContainerCmd;import com.github.dockerjava.api.command.WaitContainerCmd;import com.github.dockerjava.api.exception.DockerException;import com.github.dockerjava.api.exception.NotFoundException;@@ -18,6 +19,8 @@ import org.junit.jupiter.api.extension.ExtendWith;import org.mockito.Mock;import org.mockito.junit.jupiter.MockitoExtension;+import java.util.ArrayList;+import java.util.List;import java.util.Optional;import java.util.OptionalInt;@@ -46,6 +49,49 @@ class ContainerLifecycleManagerVolumeTest {manager = new ContainerLifecycleManager(dockerClient, imageCacheService, containerDetector, portAllocator);}+ @Test+ void stopAndRemoveStopsContainerBeforeClosingItsLogStream() {+ StopContainerCmd stop = mock(StopContainerCmd.class);+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class);+ List<String> operations = new ArrayList<>();+ doAnswer(invocation -> {+ operations.add("stop");+ return null;+ }).when(stop).exec();+ doAnswer(invocation -> {+ operations.add("remove");+ return null;+ }).when(remove).exec();+ when(dockerClient.stopContainerCmd("container-id")).thenReturn(stop);+ when(stop.withTimeout(5)).thenReturn(stop);+ when(dockerClient.removeContainerCmd("container-id")).thenReturn(remove);+ when(remove.withForce(true)).thenReturn(remove);++ manager.stopAndRemove("container-id", () -> operations.add("logs"));++ assertEquals(List.of("stop", "remove", "logs"), operations);+ }++ @Test+ void stopAndRemoveForceRemovesAndFinalizesLogStreamWhenContainerStopFails() {+ StopContainerCmd stop = mock(StopContainerCmd.class);+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class);+ List<String> operations = new ArrayList<>();+ doThrow(new RuntimeException("Docker daemon unavailable")).when(stop).exec();+ doAnswer(invocation -> {+ operations.add("remove");+ return null;+ }).when(remove).exec();+ when(dockerClient.stopContainerCmd("container-id")).thenReturn(stop);+ when(stop.withTimeout(5)).thenReturn(stop);+ when(dockerClient.removeContainerCmd("container-id")).thenReturn(remove);+ when(remove.withForce(true)).thenReturn(remove);++ manager.stopAndRemove("container-id", () -> operations.add("logs"));++ assertEquals(List.of("remove", "logs"), operations);+ }+@Testvoid volumeExists_returnsTrue_whenVolumeExists() {InspectVolumeCmd cmd = mock(InspectVolumeCmd.class);@@ -218,4 +264,3 @@ class ContainerLifecycleManagerVolumeTest {verify(dockerClient, times(2)).createContainerCmd("busybox:stable");}}-diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.javanew file mode 100644index 0000000000..04a76307cd--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java@@ -0,0 +1,328 @@+package io.github.hectorvent.floci.core.common.docker;++import com.github.dockerjava.api.model.Frame;+import com.github.dockerjava.api.model.StreamType;+import org.junit.jupiter.api.Test;++import java.io.Closeable;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.Collections;+import java.util.List;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;++class ContainerLogStreamerTest {++ private static byte[] utf8(String s) {+ return s.getBytes(StandardCharsets.UTF_8);+ }++ @Test+ void reassemblesSingleLineSplitAcrossFrames() {+ // A ~1.5 KB stdout line that Docker delivers as multiple frames must arrive+ // as ONE CloudWatch event, not several (regression for the >1024-byte split).+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();+ String longLine = "x".repeat(1500);++ assertEquals(List.of(), buffer.append(utf8(longLine.substring(0, 1024))));+ assertEquals(List.of(), buffer.append(utf8(longLine.substring(1024))));++ List<String> lines = buffer.append(utf8("\n"));+ assertEquals(1, lines.size());+ assertEquals(1500, lines.get(0).length());+ assertEquals(longLine, lines.get(0));+ }++ @Test+ void splitsMultipleLinesPackedIntoOneFrame() {+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();++ assertEquals(List.of("first", "second", "third"), buffer.append(utf8("first\nsecond\nthird\n")));+ }++ @Test+ void flushReturnsTrailingUnterminatedContent() {+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();++ assertEquals(List.of("complete"), buffer.append(utf8("complete\npartial")));+ assertEquals("partial", buffer.flush().orElseThrow());+ assertTrue(buffer.flush().isEmpty(), "buffer should be cleared after flush");+ }++ @Test+ void doesNotCorruptMultibyteCharacterSplitAcrossFrames() {+ // "€" is E2 82 AC in UTF-8; decoding per-frame would corrupt it, decoding the+ // reassembled line does not.+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();+ byte[] euro = utf8("€");++ assertEquals(List.of(), buffer.append(new byte[]{euro[0], euro[1]}));+ assertEquals(List.of("€"), buffer.append(new byte[]{euro[2], '\n'}));+ }++ @Test+ void preservesEmptyLinesInSplitOutput() {+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();++ // append() splits faithfully; the emit layer is what skips blanks.+ assertEquals(List.of("a", "", "b"), buffer.append(utf8("a\n\nb\n")));+ }++ @Test+ void boundsBufferBySplittingOverlongUnterminatedRuns() {+ // A 256 KB run with no newline must be force-emitted (bounded), not buffered until the+ // heap is exhausted — regression guard for the unbounded-growth review finding.+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();+ byte[] overlong = new byte[256 * 1024];+ java.util.Arrays.fill(overlong, (byte) 'a');++ List<String> lines = buffer.append(overlong);++ assertEquals(1, lines.size());+ assertEquals(256 * 1024, lines.get(0).length());+ assertTrue(buffer.flush().isEmpty(), "buffer is drained after the forced split");+ }++ @Test+ void forcedSplitKeepsEventWithinCapAndDefersStraddlingMultibyteChar() {+ // Regression for the oversized-event finding: with the buffer one byte below the cap, a 3-byte+ // '€' would inflate the event to cap+2. The split must instead emit the buffered run WITHIN the+ // cap and defer the whole '€' to the next event — never cutting the character.+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();+ int cap = 256 * 1024;+ byte[] fill = new byte[cap - 1];+ java.util.Arrays.fill(fill, (byte) 'a');+ assertEquals(List.of(), buffer.append(fill));++ List<String> lines = buffer.append(utf8("€"));++ assertEquals(1, lines.size());+ String event = lines.get(0);+ assertTrue(event.getBytes(StandardCharsets.UTF_8).length <= cap, "event must not exceed the cap");+ assertFalse(event.contains("�"), "no half-decoded character in the emitted event");+ assertTrue(event.chars().allMatch(c -> c == 'a'), "the straddling '€' was deferred, not included");+ // '€' is buffered whole and emitted intact on the next newline.+ assertEquals("€", buffer.append(utf8("\n")).get(0));+ }++ @Test+ void keepsCharacterIntactWhenItsFinalBytesArriveAtTheCapInAnotherFrame() {+ // A four-byte character that exactly fills the remaining capacity must not be decoded after its+ // first frame. The terminal continuation bytes complete the same capped event intact.+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();+ int cap = 256 * 1024;+ byte[] fill = new byte[cap - 4];+ java.util.Arrays.fill(fill, (byte) 'a');+ byte[] rocket = utf8("🚀");++ assertEquals(List.of(), buffer.append(fill));+ assertEquals(List.of(), buffer.append(new byte[]{rocket[0], rocket[1]}));+ List<String> lines = buffer.append(new byte[]{rocket[2], rocket[3]});++ assertEquals(1, lines.size());+ assertTrue(lines.get(0).endsWith("🚀"));+ assertFalse(lines.get(0).contains("�"));+ }++ @Test+ void boundsBufferOnRepeatedLeadingBytesThatNeverCompleteACharacter() {+ // Invalid UTF-8 that never reaches a character boundary (a run of lead bytes) must still be+ // force-split at the hard ceiling, not buffered without bound.+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();+ byte[] leads = new byte[256 * 1024 + 64];+ java.util.Arrays.fill(leads, (byte) 0xE2); // 1110xxxx lead byte, never followed by continuations+ List<String> lines = buffer.append(leads);+ assertFalse(lines.isEmpty(), "a boundary-less lead-byte run must be force-split, not buffered unbounded");+ }++ @Test+ void boundsBufferOnStrayContinuationBytes() {+ // A run of stray continuation bytes (invalid UTF-8, no lead byte) must be force-split at the+ // cap rather than growing unbounded.+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();+ byte[] continuations = new byte[256 * 1024 + 64];+ java.util.Arrays.fill(continuations, (byte) 0x80); // 10xxxxxx continuation+ List<String> lines = buffer.append(continuations);+ assertFalse(lines.isEmpty(), "a continuation-only run must be force-split, not buffered unbounded");+ }++ @Test+ void keepsLateUnterminatedFragmentsTogetherUntilTerminal() throws Exception {+ // A cancelled transport can still deliver an in-flight continuation. It must not flush the first+ // fragment then let that continuation become a second event. The terminal callback, not elapsed+ // time, is the boundary that flushes the complete logical line.+ List<String> emitted = new ArrayList<>();+ ContainerLogStreamer.LogReassemblyCallback cb =+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add);++ cb.onNext(new Frame(StreamType.STDOUT, utf8("partial")));+ cb.close();+ cb.onNext(new Frame(StreamType.STDOUT, utf8("-late")));+ assertEquals(List.of(), emitted, "the partial line stays intact until the reader terminates");+ cb.onComplete();++ assertEquals(List.of("partial-late"), emitted);+ }++ @Test+ void lifecycleHandleCloseRetainsDelayedFramesUntilTerminal() throws Exception {+ // Container shutdown has completed, but the Docker reader can still deliver bytes it already read.+ // Closing the lifecycle handle must keep the reader attached through its drain period so a delayed+ // continuation reaches the terminal flush rather than being truncated immediately.+ List<String> emitted = Collections.synchronizedList(new ArrayList<>());+ ContainerLogStreamer.LogReassemblyCallback cb =+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add);+ Closeable handle = new ContainerLogStreamer.ContainerLogHandle(cb);+ cb.onNext(new Frame(StreamType.STDOUT, utf8("before")));++ handle.close();+ cb.onNext(new Frame(StreamType.STDOUT, utf8("-after")));+ cb.onComplete();++ assertEquals(List.of("before-after"), emitted);+ }++ @Test+ void emitsCompletedLinesBeforeTheFinalTailDuringConcurrentLifecycleClose() throws Exception {+ // The reader begins emitting a complete line, then a lifecycle-confirmed container stop drains a+ // trailing partial line. The final tail must queue behind the already-complete line, even though+ // the consumer for that line is still running on a different thread.+ List<String> emitted = Collections.synchronizedList(new ArrayList<>());+ CountDownLatch firstEmissionStarted = new CountDownLatch(1);+ CountDownLatch releaseFirstEmission = new CountDownLatch(1);+ ContainerLogStreamer.LogReassemblyCallback cb =+ new ContainerLogStreamer.LogReassemblyCallback(line -> {+ if ("complete".equals(line)) {+ firstEmissionStarted.countDown();+ try {+ releaseFirstEmission.await();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AssertionError(e);+ }+ }+ emitted.add(line);+ });++ Thread reader = new Thread(() -> cb.onNext(new Frame(StreamType.STDOUT, utf8("complete\npartial"))));+ reader.start();+ assertTrue(firstEmissionStarted.await(1, TimeUnit.SECONDS), "the first line should begin emitting");++ Thread closer = new Thread(() -> {+ cb.closeAfterContainerStopped();+ });+ closer.start();+ Thread terminal = new Thread(cb::onComplete);+ terminal.start();+ closer.join();+ releaseFirstEmission.countDown();+ reader.join();+ terminal.join();++ assertEquals(List.of("complete", "partial"), emitted);+ }++ @Test+ void framesDeliveredAfterTransportCloseAreStillEmitted() throws Exception {+ // Cancelling the transport must not permanently stop buffering. A frame the reader delivers after+ // cancellation is still emitted; the eventual terminal drains the rest.+ List<String> emitted = new ArrayList<>();+ ContainerLogStreamer.LogReassemblyCallback cb =+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add);++ cb.onNext(new Frame(StreamType.STDOUT, utf8("before-grace")));+ cb.close();+ cb.onNext(new Frame(StreamType.STDOUT, utf8("-after-close")));+ cb.onComplete(); // the reader's real terminal drains the reassembled tail++ assertEquals(List.of("before-grace-after-close"), emitted);+ }++ @Test+ void onNextAfterTerminalIsIgnored() throws Exception {+ // A straggler frame arriving after the reader's TERMINAL (the true end of the stream) must not be+ // appended into a buffer that will never be flushed — the terminated flag, checked under the+ // lock, drops it deterministically.+ List<String> emitted = new ArrayList<>();+ ContainerLogStreamer.LogReassemblyCallback cb =+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add);++ cb.onNext(new Frame(StreamType.STDOUT, utf8("kept\n")));+ cb.onComplete(); // true terminal: drains and marks terminated+ cb.onNext(new Frame(StreamType.STDOUT, utf8("straggler-after-terminal")));++ assertEquals(List.of("kept"), emitted);+ }++ @Test+ void reassemblesInFlightFramesDeliveredDuringClose() throws Exception {+ // Fragments of ONE newline-free write are delivered by the transport reader AFTER close() is+ // called; the terminal callback then flushes their reassembled line. A separate thread supplies+ // the reader callbacks, mirroring the transport.+ List<String> emitted = java.util.Collections.synchronizedList(new ArrayList<>());+ ContainerLogStreamer.LogReassemblyCallback cb =+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add);++ cb.onNext(new Frame(StreamType.STDOUT, utf8("frag1")));++ Thread reader = new Thread(() -> {+ cb.onNext(new Frame(StreamType.STDOUT, utf8("frag2")));+ cb.onNext(new Frame(StreamType.STDOUT, utf8("frag3\n")));+ cb.onComplete(); // terminal: drains and unblocks close()+ });+ reader.start();++ cb.close();+ reader.join();++ assertEquals(List.of("frag1frag2frag3"), emitted);+ }++ @Test+ void flushesTrailingContentOnComplete() throws Exception {+ // A clean end (onComplete) flushes any trailing unterminated line.+ List<String> emitted = new ArrayList<>();+ ContainerLogStreamer.LogReassemblyCallback cb =+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add);++ cb.onNext(new Frame(StreamType.STDOUT, utf8("no-newline-tail")));+ cb.onComplete();++ assertEquals(List.of("no-newline-tail"), emitted);+ }++ @Test+ void flushesStreamTailsInTheirLastFrameArrivalOrder() {+ List<String> emitted = new ArrayList<>();+ ContainerLogStreamer.LogReassemblyCallback cb =+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add);++ cb.onNext(new Frame(StreamType.STDERR, utf8("stderr-tail")));+ cb.onNext(new Frame(StreamType.STDOUT, utf8("stdout-tail")));+ cb.onComplete();++ assertEquals(List.of("stderr-tail", "stdout-tail"), emitted);+ }++ @Test+ void lifecycleHandleFlushesTailWhenDockerNeverSignalsTerminal() throws Exception {+ List<String> emitted = Collections.synchronizedList(new ArrayList<>());+ ContainerLogStreamer.LogReassemblyCallback cb =+ new ContainerLogStreamer.LogReassemblyCallback(emitted::add, 10);+ Closeable handle = new ContainerLogStreamer.ContainerLogHandle(cb);++ cb.onNext(new Frame(StreamType.STDOUT, utf8("unterminated-tail")));+ handle.close();++ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1);+ while (emitted.isEmpty() && System.nanoTime() < deadline) {+ Thread.sleep(5);+ }+ assertEquals(List.of("unterminated-tail"), emitted);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildServiceTest.javanew file mode 100644index 0000000000..f869a93bb9--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildServiceTest.java@@ -0,0 +1,51 @@+package io.github.hectorvent.floci.services.codebuild;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.services.codebuild.model.Build;+import io.github.hectorvent.floci.services.codebuild.model.Project;+import io.github.hectorvent.floci.services.codebuild.model.ProjectArtifacts;+import io.github.hectorvent.floci.services.codebuild.model.ProjectEnvironment;+import io.github.hectorvent.floci.services.codebuild.model.ProjectSource;+import org.junit.jupiter.api.Test;+import org.mockito.ArgumentCaptor;++import java.util.List;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;++class CodeBuildServiceTest {++ private static final String REGION = "us-east-1";+ private static final String ACCOUNT = "000000000000";++ @Test+ void retryBuildRetainsOriginalBuildspecOverride() {+ CodeBuildRunner runner = mock(CodeBuildRunner.class);+ CodeBuildService service = new CodeBuildService(runner, mock(EmulatorConfig.class), null);+ ProjectSource source = new ProjectSource();+ source.setType("NO_SOURCE");+ ProjectArtifacts artifacts = new ProjectArtifacts();+ artifacts.setType("NO_ARTIFACTS");++ service.createProject(REGION, ACCOUNT, "retry-project", null, source, null, null,+ artifacts, null, new ProjectEnvironment(), "arn:aws:iam::000000000000:role/codebuild",+ null, null, null, null, null, null, null);++ String buildspec = "version: 0.2\nphases:\n build:\n commands:\n - echo retry\n";+ Build original = service.startBuild(REGION, ACCOUNT, "retry-project", buildspec,+ null, null, null, null, null, null);+ Build retried = service.retryBuild(REGION, ACCOUNT, original.getId());++ assertNotEquals(original.getId(), retried.getId());+ ArgumentCaptor<String> buildspecOverrides = ArgumentCaptor.forClass(String.class);+ verify(runner, times(2)).startBuild(eq(REGION), any(Build.class), any(Project.class),+ buildspecOverrides.capture());+ assertEquals(List.of(buildspec, buildspec), buildspecOverrides.getAllValues());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.javaindex b1d93235f4..bcce5ad76b 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/EcsServiceTeardownTest.java@@ -12,9 +12,11 @@ import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;import io.github.hectorvent.floci.services.ecs.model.LaunchType;import org.junit.jupiter.api.Test;+import java.io.Closeable;import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.concurrent.atomic.AtomicInteger;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;@@ -72,6 +74,50 @@ class EcsServiceTeardownTest {verify(containerManager, times(1)).stopTask(handle);}+ @Test+ void stoppedTaskRetriesTeardownUntilItsRemainingLogStreamIsReleased() {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().ecs().mock()).thenReturn(false); // docker mode+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");++ EcsContainerManager containerManager = mock(EcsContainerManager.class);+ EcsTaskHandle handle = new EcsTaskHandle("task-arn", Map.of("app", "docker-id"),+ Map.of("docker-id", mock(Closeable.class)));+ when(containerManager.startTask(any(), any(), any(), anyString())).thenReturn(handle);+ AtomicInteger teardownAttempts = new AtomicInteger();+ when(containerManager.stopTaskAndCollectExitCodes(handle)).thenAnswer(ignored -> {+ if (teardownAttempts.incrementAndGet() == 2) {+ handle.removeLogStream("docker-id");+ }+ return Map.of();+ });++ EcsService service = new EcsService(+ new RegionResolver(REGION, "000000000000"),+ containerManager,+ config,+ mock(EcsLoadBalancerRegistrar.class),+ new SingleUseStorageFactory());+ service.initializeStorage();++ ContainerDefinition cd = new ContainerDefinition();+ cd.setName("app");+ cd.setImage("nginx:alpine");+ service.registerTaskDefinition("retry-fam", List.of(cd), null, null, null,+ null, null, List.of(), REGION);+ String taskArn = service.runTask(null, "retry-fam", 1, LaunchType.FARGATE, null, null,+ List.of(), null, REGION).getFirst().getTaskArn();++ service.stopTask(null, taskArn, null, REGION);+ verify(containerManager, times(1)).stopTaskAndCollectExitCodes(handle);++ service.reconcile();+ verify(containerManager, times(2)).stopTaskAndCollectExitCodes(handle);++ service.reconcile();+ verify(containerManager, times(2)).stopTaskAndCollectExitCodes(handle);+ }+private static final class SingleUseStorageFactory extends StorageFactory {private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();diff --git a/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.javanew file mode 100644index 0000000000..928c0f0b21--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManagerLifecycleTest.java@@ -0,0 +1,121 @@+package io.github.hectorvent.floci.services.ecs.container;++import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.RemoveContainerCmd;+import com.github.dockerjava.api.command.StopContainerCmd;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;+import io.github.hectorvent.floci.services.ssm.SsmService;+import org.junit.jupiter.api.Test;++import java.io.Closeable;+import java.util.LinkedHashMap;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertSame;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.doThrow;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++class EcsContainerManagerLifecycleTest {++ @Test+ void finalizesTaskLogStreamsAfterForceRemovingAContainerWhoseStopFails() {+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ DockerClient dockerClient = mock(DockerClient.class);+ StopContainerCmd stop = mock(StopContainerCmd.class);+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class);+ Closeable logStream = mock(Closeable.class);+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);+ when(dockerClient.stopContainerCmd("docker-id")).thenReturn(stop);+ when(stop.withTimeout(5)).thenReturn(stop);+ when(dockerClient.removeContainerCmd("docker-id")).thenReturn(remove);+ when(remove.withForce(true)).thenReturn(remove);+ doThrow(new RuntimeException("Docker daemon unavailable")).when(stop).exec();++ EcsContainerManager manager = manager(lifecycleManager);+ EcsTaskHandle handle = new EcsTaskHandle("task-arn", Map.of("app", "docker-id"),+ Map.of("docker-id", logStream));++ manager.stopTaskAndCollectExitCodes(handle);++ verify(remove).exec();+ verify(lifecycleManager).closeLogStreamAfterContainerStop(logStream);+ }++ @Test+ void finalizesTaskLogStreamsAfterEveryContainerStops() {+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ DockerClient dockerClient = mock(DockerClient.class);+ StopContainerCmd stop = mock(StopContainerCmd.class);+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class);+ Closeable logStream = mock(Closeable.class);+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);+ when(dockerClient.stopContainerCmd("docker-id")).thenReturn(stop);+ when(stop.withTimeout(5)).thenReturn(stop);+ when(dockerClient.removeContainerCmd("docker-id")).thenReturn(remove);+ when(remove.withForce(true)).thenReturn(remove);++ EcsContainerManager manager = manager(lifecycleManager);+ EcsTaskHandle handle = new EcsTaskHandle("task-arn", Map.of("app", "docker-id"),+ Map.of("docker-id", logStream));++ manager.stopTaskAndCollectExitCodes(handle);++ verify(lifecycleManager).closeLogStreamAfterContainerStop(logStream);+ }++ @Test+ void preservesOnlyTheLogStreamForAContainerThatCouldNotStopOrBeRemoved() {+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ DockerClient dockerClient = mock(DockerClient.class);+ StopContainerCmd failedStop = mock(StopContainerCmd.class);+ StopContainerCmd successfulStop = mock(StopContainerCmd.class);+ RemoveContainerCmd failedRemove = mock(RemoveContainerCmd.class);+ RemoveContainerCmd successfulRemove = mock(RemoveContainerCmd.class);+ Closeable retainedLogStream = mock(Closeable.class);+ Closeable finalizedLogStream = mock(Closeable.class);+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);+ when(dockerClient.stopContainerCmd("running-id")).thenReturn(failedStop);+ when(failedStop.withTimeout(5)).thenReturn(failedStop);+ doThrow(new RuntimeException("stop failed")).when(failedStop).exec();+ when(dockerClient.stopContainerCmd("stopped-id")).thenReturn(successfulStop);+ when(successfulStop.withTimeout(5)).thenReturn(successfulStop);+ when(dockerClient.removeContainerCmd("running-id")).thenReturn(failedRemove);+ when(failedRemove.withForce(true)).thenReturn(failedRemove);+ doThrow(new RuntimeException("remove failed")).when(failedRemove).exec();+ when(dockerClient.removeContainerCmd("stopped-id")).thenReturn(successfulRemove);+ when(successfulRemove.withForce(true)).thenReturn(successfulRemove);++ Map<String, String> containerIds = new LinkedHashMap<>();+ containerIds.put("running", "running-id");+ containerIds.put("stopped", "stopped-id");+ EcsTaskHandle handle = new EcsTaskHandle("task-arn", containerIds,+ Map.of("running-id", retainedLogStream, "stopped-id", finalizedLogStream));++ manager(lifecycleManager).stopTaskAndCollectExitCodes(handle);++ verify(lifecycleManager, never()).closeLogStreamAfterContainerStop(retainedLogStream);+ verify(lifecycleManager).closeLogStreamAfterContainerStop(finalizedLogStream);+ assertTrue(handle.hasOpenLogStreams());+ assertSame(retainedLogStream, handle.getLogStreamsByContainerId().get("running-id"));+ assertFalse(handle.getLogStreamsByContainerId().containsKey("stopped-id"));+ }++ private static EcsContainerManager manager(ContainerLifecycleManager lifecycleManager) {+ return new EcsContainerManager(+ mock(ContainerBuilder.class), lifecycleManager, mock(ContainerLogStreamer.class),+ mock(ContainerDetector.class), mock(EmulatorConfig.class), mock(RegionResolver.class),+ mock(LaunchedContainerAwsEnv.class), mock(SsmService.class), mock(SecretsManagerService.class));+ }+}