Light, fluffy, and always free - The AWS Local Emulator alternative
fix: add log rotation by default to all floci-launched containers (#466)
Conor Heinec0e4af32f710
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 1a5ce8d17b..1c9b655506 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -53,6 +53,8 @@ public interface EmulatorConfig {ServicesConfig services();+ DockerConfig docker();+InitHooksConfig initHooks();interface StorageConfig {@@ -576,4 +578,24 @@ public interface EmulatorConfig {@WithDefault("30")long timeoutSeconds();}++ /**+ * Configuration for Docker container management shared across all services+ * that spawn Docker containers (Lambda, RDS, ElastiCache, ECS, ECR, MSK).+ */+ interface DockerConfig {+ /**+ * Maximum size of each container log file before rotation.+ * Uses Docker's json-file log driver max-size option format (e.g., "10m", "100k", "1g").+ */+ @WithDefault("10m")+ String logMaxSize();++ /**+ * Maximum number of rotated log files to retain per container.+ * When this limit is reached, the oldest log file is deleted.+ */+ @WithDefault("3")+ String logMaxFile();+ }}diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.javanew file mode 100644index 0000000000..0e34d17944--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerBuilder.java@@ -0,0 +1,292 @@+package io.github.hectorvent.floci.core.common.docker;++import io.github.hectorvent.floci.config.EmulatorConfig;+import com.github.dockerjava.api.model.Bind;+import com.github.dockerjava.api.model.LogConfig;+import com.github.dockerjava.api.model.Mount;+import com.github.dockerjava.api.model.MountType;+import com.github.dockerjava.api.model.Volume;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;++import java.util.ArrayList;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.Optional;++/**+ * Fluent builder for constructing {@link ContainerSpec} instances.+ * Provides sensible defaults and integrates with Floci configuration.+ *+ * <p>Example usage:+ * <pre>{@code+ * ContainerSpec spec = containerBuilder.newContainer("nginx:latest")+ * .withName("floci-my-service")+ * .withEnv("MY_VAR", "value")+ * .withDynamicPort(8080)+ * .withDockerNetwork(config.services().myService().dockerNetwork())+ * .withLogRotation()+ * .build();+ * }</pre>+ */+@ApplicationScoped+public class ContainerBuilder {++ private final EmulatorConfig config;+ private final DockerHostResolver dockerHostResolver;++ @Inject+ public ContainerBuilder(EmulatorConfig config, DockerHostResolver dockerHostResolver) {+ this.config = config;+ this.dockerHostResolver = dockerHostResolver;+ }++ /**+ * Creates a new builder for a container with the specified image.+ *+ * @param image Docker image name (e.g., "nginx:latest")+ * @return a new Builder instance+ */+ public Builder newContainer(String image) {+ return new Builder(image, config, dockerHostResolver);+ }++ /**+ * Fluent builder for constructing ContainerSpec instances.+ */+ public static class Builder {+ private final String image;+ private final EmulatorConfig config;+ private final DockerHostResolver dockerHostResolver;++ private String name;+ private final List<String> env = new ArrayList<>();+ private List<String> cmd;+ private List<String> entrypoint;+ private Long memoryBytes;+ private final Map<Integer, Integer> portBindings = new HashMap<>();+ private final List<Integer> exposedPorts = new ArrayList<>();+ private String networkMode;+ private final List<Mount> mounts = new ArrayList<>();+ private final List<Bind> binds = new ArrayList<>();+ private final List<String> extraHosts = new ArrayList<>();+ private LogConfig logConfig;++ Builder(String image, EmulatorConfig config, DockerHostResolver dockerHostResolver) {+ this.image = image;+ this.config = config;+ this.dockerHostResolver = dockerHostResolver;+ }++ /**+ * Sets the container name.+ */+ public Builder withName(String name) {+ this.name = name;+ return this;+ }++ /**+ * Adds a single environment variable.+ */+ public Builder withEnv(String key, String value) {+ this.env.add(key + "=" + value);+ return this;+ }++ /**+ * Adds multiple environment variables from a list of "KEY=value" strings.+ */+ public Builder withEnv(List<String> env) {+ this.env.addAll(env);+ return this;+ }++ /**+ * Sets the container command (overrides image CMD).+ */+ public Builder withCmd(List<String> cmd) {+ this.cmd = cmd;+ return this;+ }++ /**+ * Sets the container command from a single string (for simple commands).+ */+ public Builder withCmd(String cmd) {+ this.cmd = List.of(cmd);+ return this;+ }++ /**+ * Sets the container entrypoint (overrides image ENTRYPOINT).+ */+ public Builder withEntrypoint(List<String> entrypoint) {+ this.entrypoint = entrypoint;+ return this;+ }++ /**+ * Sets the memory limit in megabytes.+ */+ public Builder withMemoryMb(int memoryMb) {+ this.memoryBytes = (long) memoryMb * 1024 * 1024;+ return this;+ }++ /**+ * Sets the memory limit in bytes.+ */+ public Builder withMemoryBytes(long memoryBytes) {+ this.memoryBytes = memoryBytes;+ return this;+ }++ /**+ * Adds a port binding from container port to a specific host port.+ */+ public Builder withPortBinding(int containerPort, int hostPort) {+ this.portBindings.put(containerPort, hostPort);+ this.exposedPorts.add(containerPort);+ return this;+ }++ /**+ * Adds a port binding with dynamic host port allocation.+ * Use this when you don't care which host port is used.+ */+ public Builder withDynamicPort(int containerPort) {+ return withPortBinding(containerPort, 0);+ }++ /**+ * Exposes a port without creating a host binding.+ * Useful when containers communicate via Docker network.+ */+ public Builder withExposedPort(int port) {+ this.exposedPorts.add(port);+ return this;+ }++ /**+ * Sets the Docker network mode directly.+ */+ public Builder withNetworkMode(String networkMode) {+ this.networkMode = networkMode;+ return this;+ }++ /**+ * Sets the Docker network from a service-specific Optional, falling back to+ * the global services.dockerNetwork() if not present.+ * This is the standard pattern for Floci container services.+ */+ public Builder withDockerNetwork(Optional<String> serviceNetwork) {+ serviceNetwork+ .or(() -> config.services().dockerNetwork())+ .filter(n -> !n.isBlank())+ .ifPresent(n -> this.networkMode = n);+ return this;+ }++ /**+ * Adds a bind mount from host path to container path.+ */+ public Builder withBind(String hostPath, String containerPath) {+ this.binds.add(new Bind(hostPath, new Volume(containerPath)));+ return this;+ }++ /**+ * Adds a named volume mount.+ */+ public Builder withNamedVolume(String volumeName, String containerPath) {+ this.mounts.add(new Mount()+ .withType(MountType.VOLUME)+ .withSource(volumeName)+ .withTarget(containerPath));+ return this;+ }++ /**+ * Adds a mount (any type: volume, bind, tmpfs).+ */+ public Builder withMount(Mount mount) {+ this.mounts.add(mount);+ return this;+ }++ /**+ * Adds the host.docker.internal extra host entry on Linux.+ * This allows containers to reach the host via a consistent hostname+ * across all platforms (already exists on Docker Desktop).+ */+ public Builder withHostDockerInternalOnLinux() {+ if (dockerHostResolver.isLinuxHost()) {+ this.extraHosts.add("host.docker.internal:host-gateway");+ }+ return this;+ }++ /**+ * Adds a custom extra host entry.+ */+ public Builder withExtraHost(String hostname, String ip) {+ this.extraHosts.add(hostname + ":" + ip);+ return this;+ }++ /**+ * Enables log rotation with default settings from configuration.+ * Uses json-file driver with max-size and max-file from config.+ */+ public Builder withLogRotation() {+ String maxSize = config.docker().logMaxSize();+ String maxFile = config.docker().logMaxFile();+ return withLogRotation(maxSize, maxFile);+ }++ /**+ * Enables log rotation with custom settings.+ *+ * @param maxSize maximum log file size (e.g., "10m", "100k", "1g")+ * @param maxFile maximum number of log files to retain+ */+ public Builder withLogRotation(String maxSize, String maxFile) {+ this.logConfig = new LogConfig(+ LogConfig.LoggingType.JSON_FILE,+ Map.of("max-size", maxSize, "max-file", maxFile));+ return this;+ }++ /**+ * Sets a custom log configuration.+ */+ public Builder withLogConfig(LogConfig logConfig) {+ this.logConfig = logConfig;+ return this;+ }++ /**+ * Builds the immutable ContainerSpec.+ */+ public ContainerSpec build() {+ return new ContainerSpec(+ image,+ name,+ List.copyOf(env),+ cmd != null ? List.copyOf(cmd) : null,+ entrypoint != null ? List.copyOf(entrypoint) : null,+ memoryBytes,+ Map.copyOf(portBindings),+ List.copyOf(exposedPorts),+ networkMode,+ List.copyOf(mounts),+ List.copyOf(binds),+ List.copyOf(extraHosts),+ logConfig+ );+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javanew file mode 100644index 0000000000..26cf66bc47--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -0,0 +1,375 @@+package io.github.hectorvent.floci.core.common.docker;++import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;+import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.CreateContainerCmd;+import com.github.dockerjava.api.command.CreateContainerResponse;+import com.github.dockerjava.api.command.InspectContainerResponse;+import com.github.dockerjava.api.exception.NotFoundException;+import com.github.dockerjava.api.model.Bind;+import com.github.dockerjava.api.model.Container;+import com.github.dockerjava.api.model.ContainerNetwork;+import com.github.dockerjava.api.model.ExposedPort;+import com.github.dockerjava.api.model.HostConfig;+import com.github.dockerjava.api.model.Ports;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.io.Closeable;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.Optional;++/**+ * Manages Docker container lifecycle operations including create, start, stop, and remove.+ * Consolidates common container management patterns used across Floci services.+ */+@ApplicationScoped+public class ContainerLifecycleManager {++ private static final Logger LOG = Logger.getLogger(ContainerLifecycleManager.class);++ private final DockerClient dockerClient;+ private final ImageCacheService imageCacheService;+ private final ContainerDetector containerDetector;+ private final PortAllocator portAllocator;++ @Inject+ public ContainerLifecycleManager(DockerClient dockerClient,+ ImageCacheService imageCacheService,+ ContainerDetector containerDetector,+ PortAllocator portAllocator) {+ this.dockerClient = dockerClient;+ this.imageCacheService = imageCacheService;+ this.containerDetector = containerDetector;+ this.portAllocator = portAllocator;+ }++ /**+ * Creates and starts a container from the given specification.+ * Automatically pulls the image if not present locally.+ *+ * @param spec the container specification+ * @return information about the created container including resolved endpoints+ */+ public ContainerInfo createAndStart(ContainerSpec spec) {+ LOG.debugv("Creating container from spec: image={0}, name={1}", spec.image(), spec.name());++ // Ensure image is available+ imageCacheService.ensureImageExists(spec.image());++ // Build HostConfig+ HostConfig hostConfig = buildHostConfig(spec);++ // Create container command+ CreateContainerCmd createCmd = dockerClient.createContainerCmd(spec.image())+ .withHostConfig(hostConfig);++ if (spec.name() != null) {+ createCmd.withName(spec.name());+ }+ if (spec.env() != null && !spec.env().isEmpty()) {+ createCmd.withEnv(spec.env());+ }+ if (spec.cmd() != null && !spec.cmd().isEmpty()) {+ createCmd.withCmd(spec.cmd());+ }+ if (spec.entrypoint() != null && !spec.entrypoint().isEmpty()) {+ createCmd.withEntrypoint(spec.entrypoint());+ }+ if (spec.exposedPorts() != null && !spec.exposedPorts().isEmpty()) {+ ExposedPort[] exposed = spec.exposedPorts().stream()+ .map(ExposedPort::tcp)+ .toArray(ExposedPort[]::new);+ createCmd.withExposedPorts(exposed);+ }++ // Create the container+ CreateContainerResponse response = createCmd.exec();+ String containerId = response.getId();+ LOG.infov("Created container {0} (name={1})", containerId, spec.name());++ // Start the container+ dockerClient.startContainerCmd(containerId).exec();+ LOG.infov("Started container {0}", containerId);++ // Resolve endpoints+ Map<Integer, EndpointInfo> endpoints = resolveEndpoints(containerId, spec);++ return new ContainerInfo(containerId, endpoints);+ }++ /**+ * Stops and removes a container, closing any associated log stream.+ *+ * @param containerId the container ID to stop and remove+ * @param logStream optional log stream to close (may be null)+ */+ public void stopAndRemove(String containerId, Closeable logStream) {+ LOG.infov("Stopping container {0}", containerId);++ // Close log stream first+ if (logStream != null) {+ try {+ logStream.close();+ } catch (Exception e) {+ LOG.debugv("Error closing log stream: {0}", e.getMessage());+ }+ }++ // Stop container+ try {+ dockerClient.stopContainerCmd(containerId).withTimeout(5).exec();+ } catch (NotFoundException e) {+ LOG.debugv("Container {0} not found (already removed)", containerId);+ return;+ } catch (Exception e) {+ LOG.warnv("Error stopping container {0}: {1}", containerId, e.getMessage());+ }++ // Remove container+ try {+ dockerClient.removeContainerCmd(containerId).withForce(true).exec();+ LOG.debugv("Removed container {0}", containerId);+ } catch (NotFoundException e) {+ // Already gone+ } catch (Exception e) {+ LOG.warnv("Error removing container {0}: {1}", containerId, e.getMessage());+ }+ }++ /**+ * Finds an existing container by name.+ *+ * @param name the container name to search for+ * @return the container if found+ */+ public Optional<Container> findByName(String name) {+ try {+ List<Container> containers = dockerClient.listContainersCmd()+ .withShowAll(true)+ .exec();++ for (Container c : containers) {+ String[] names = c.getNames();+ if (names == null) {+ continue;+ }+ for (String n : names) {+ // Docker prefixes names with /+ if (n.equals("/" + name) || n.equals(name)) {+ return Optional.of(c);+ }+ }+ }+ } catch (Exception e) {+ LOG.debugv("Error searching for container {0}: {1}", name, e.getMessage());+ }+ return Optional.empty();+ }++ /**+ * Adopts an existing container, starting it if stopped.+ * Useful for services like ECR that reuse containers across restarts.+ *+ * @param containerId the container ID to adopt+ * @param ports the container ports to resolve endpoints for+ * @return information about the adopted container+ */+ public ContainerInfo adopt(String containerId, List<Integer> ports) {+ LOG.infov("Adopting existing container {0}", containerId);++ InspectContainerResponse inspect = dockerClient.inspectContainerCmd(containerId).exec();+ boolean running = Boolean.TRUE.equals(inspect.getState().getRunning());++ if (!running) {+ dockerClient.startContainerCmd(containerId).exec();+ LOG.infov("Started adopted container {0}", containerId);+ inspect = dockerClient.inspectContainerCmd(containerId).exec();+ }++ Map<Integer, EndpointInfo> endpoints = new HashMap<>();+ for (int port : ports) {+ endpoints.put(port, resolveEndpoint(inspect, port));+ }++ return new ContainerInfo(containerId, endpoints);+ }++ /**+ * Removes a container by name if it exists. Useful for cleaning up stale containers+ * from previous runs before creating a new one.+ *+ * @param name the container name to remove+ */+ public void removeIfExists(String name) {+ try {+ dockerClient.removeContainerCmd(name).withForce(true).exec();+ LOG.infov("Removed stale container {0}", name);+ } catch (NotFoundException e) {+ // Not found - normal case+ } catch (Exception e) {+ LOG.debugv("Could not remove container {0}: {1}", name, e.getMessage());+ }+ }++ /**+ * Resolves the endpoint (host and port) to connect to a specific container port.+ *+ * @param containerId the container ID+ * @param containerPort the container port to resolve+ * @return the endpoint information+ */+ public EndpointInfo resolveEndpoint(String containerId, int containerPort) {+ InspectContainerResponse inspect = dockerClient.inspectContainerCmd(containerId).exec();+ return resolveEndpoint(inspect, containerPort);+ }++ /**+ * Returns the underlying DockerClient for operations not covered by this manager.+ * Prefer using manager methods when available.+ */+ public DockerClient getDockerClient() {+ return dockerClient;+ }++ private HostConfig buildHostConfig(ContainerSpec spec) {+ HostConfig hostConfig = HostConfig.newHostConfig();++ // Memory limit+ if (spec.hasMemoryLimit()) {+ hostConfig.withMemory(spec.memoryBytes());+ }++ // Port bindings — services decide whether to request them based on their+ // own in-container-vs-native logic. When Floci runs inside Docker, most+ // backends are reached via the docker network IP, so services omit the+ // binding. ECR's sibling registry is the exception: it always publishes+ // its port because host-side docker clients (and CDK in compat tests)+ // connect via localhost:<hostPort>.+ if (spec.hasPortBindings()) {+ Ports ports = new Ports();+ for (Map.Entry<Integer, Integer> entry : spec.portBindings().entrySet()) {+ int containerPort = entry.getKey();+ int hostPort = entry.getValue();++ // 0 means dynamic allocation+ if (hostPort == 0) {+ hostPort = portAllocator.allocateAny();+ }++ ports.bind(ExposedPort.tcp(containerPort), Ports.Binding.bindPort(hostPort));+ LOG.debugv("Port binding: {0} -> {1}", containerPort, hostPort);+ }+ hostConfig.withPortBindings(ports);+ }++ // Network mode+ if (spec.networkMode() != null && !spec.networkMode().isBlank()) {+ hostConfig.withNetworkMode(spec.networkMode());+ }++ // Mounts (named volumes, bind mounts)+ if (spec.mounts() != null && !spec.mounts().isEmpty()) {+ hostConfig.withMounts(spec.mounts());+ }++ // Legacy binds+ if (spec.binds() != null && !spec.binds().isEmpty()) {+ hostConfig.withBinds(spec.binds().toArray(new Bind[0]));+ }++ // Extra hosts (e.g., host.docker.internal on Linux)+ if (spec.extraHosts() != null && !spec.extraHosts().isEmpty()) {+ hostConfig.withExtraHosts(spec.extraHosts().toArray(new String[0]));+ }++ // Log configuration (log rotation)+ if (spec.hasLogConfig()) {+ hostConfig.withLogConfig(spec.logConfig());+ }++ return hostConfig;+ }++ private Map<Integer, EndpointInfo> resolveEndpoints(String containerId, ContainerSpec spec) {+ if (spec.exposedPorts() == null || spec.exposedPorts().isEmpty()) {+ return Map.of();+ }++ InspectContainerResponse inspect = dockerClient.inspectContainerCmd(containerId).exec();+ Map<Integer, EndpointInfo> endpoints = new HashMap<>();++ for (int containerPort : spec.exposedPorts()) {+ endpoints.put(containerPort, resolveEndpoint(inspect, containerPort));+ }++ return endpoints;+ }++ private EndpointInfo resolveEndpoint(InspectContainerResponse inspect, int containerPort) {+ if (!containerDetector.isRunningInContainer()) {+ // Native mode: use localhost and the bound host port+ var bindings = inspect.getNetworkSettings().getPorts().getBindings();+ var binding = bindings.get(ExposedPort.tcp(containerPort));++ if (binding != null && binding.length > 0) {+ int hostPort = Integer.parseInt(binding[0].getHostPortSpec());+ return new EndpointInfo("localhost", hostPort);+ }+ // Fallback to container port+ return new EndpointInfo("localhost", containerPort);+ } else {+ // Container mode: use container IP on the docker network+ String containerIp = resolveContainerIp(inspect);+ return new EndpointInfo(containerIp, containerPort);+ }+ }++ private String resolveContainerIp(InspectContainerResponse inspect) {+ var networks = inspect.getNetworkSettings().getNetworks();+ if (networks != null) {+ for (Map.Entry<String, ContainerNetwork> entry : networks.entrySet()) {+ String ip = entry.getValue().getIpAddress();+ if (ip != null && !ip.isBlank()) {+ return ip;+ }+ }+ }+ // Fallback to the global IP+ return inspect.getNetworkSettings().getIpAddress();+ }++ /**+ * Information about a created or adopted container.+ *+ * @param containerId the Docker container ID+ * @param endpoints map of container port to resolved endpoint (host:port for connection)+ */+ public record ContainerInfo(+ String containerId,+ Map<Integer, EndpointInfo> endpoints+ ) {+ /**+ * Gets the endpoint for a specific container port.+ */+ public EndpointInfo getEndpoint(int containerPort) {+ return endpoints.get(containerPort);+ }+ }++ /**+ * Network endpoint information for connecting to a container.+ *+ * @param host the host to connect to (localhost in native mode, container IP in Docker mode)+ * @param port the port to connect to+ */+ public record EndpointInfo(String host, int port) {+ @Override+ public String toString() {+ return host + ":" + port;+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.javanew file mode 100644index 0000000000..3e1d5aa973--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java@@ -0,0 +1,117 @@+package io.github.hectorvent.floci.core.common.docker;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsService;+import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.async.ResultCallback;+import com.github.dockerjava.api.model.Frame;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.io.Closeable;+import java.nio.charset.StandardCharsets;+import java.time.LocalDate;+import java.time.format.DateTimeFormatter;+import java.util.HashMap;+import java.util.List;+import java.util.Map;++/**+ * Streams Docker container logs to both the Floci console logger and CloudWatch Logs.+ * Consolidates the log streaming pattern used across container managers.+ */+@ApplicationScoped+public class ContainerLogStreamer {++ private static final Logger LOG = Logger.getLogger(ContainerLogStreamer.class);+ private static final DateTimeFormatter LOG_STREAM_DATE_FMT = DateTimeFormatter.ofPattern("yyyy/MM/dd");++ private final DockerClient dockerClient;+ private final CloudWatchLogsService cloudWatchLogsService;++ @Inject+ public ContainerLogStreamer(DockerClient dockerClient, CloudWatchLogsService cloudWatchLogsService) {+ this.dockerClient = dockerClient;+ this.cloudWatchLogsService = cloudWatchLogsService;+ }++ /**+ * Attaches a log stream to a container and forwards logs to CloudWatch Logs.+ * Returns a Closeable handle that should be closed when the container is stopped.+ *+ * @param containerId Docker container ID+ * @param logGroup CloudWatch log group name (e.g., "/aws/lambda/myFunction")+ * @param logStream CloudWatch log stream name (e.g., "2024/01/15/[$LATEST]abc123")+ * @param region AWS region for CloudWatch Logs+ * @param logPrefix prefix for console logging (e.g., "lambda:myFunction")+ * @return Closeable handle to stop the log stream+ */+ public Closeable attach(String containerId, String logGroup, String logStream,+ String region, String logPrefix) {+ ensureLogGroupAndStream(logGroup, logStream, region);++ try {+ return dockerClient.logContainerCmd(containerId)+ .withStdOut(true)+ .withStdErr(true)+ .withFollowStream(true)+ .withTimestamps(false)+ .exec(new ResultCallback.Adapter<>() {+ @Override+ public void onNext(Frame frame) {+ String line = new String(frame.getPayload(), StandardCharsets.UTF_8).stripTrailing();+ if (!line.isEmpty()) {+ LOG.infov("[{0}] {1}", logPrefix, line);+ forwardToCloudWatchLogs(logGroup, logStream, region, line);+ }+ }+ });+ } catch (Exception e) {+ LOG.warnv("Could not attach log stream for container {0}: {1}", containerId, e.getMessage());+ return null;+ }+ }++ /**+ * Creates a CloudWatch log group and stream if they don't already exist.+ */+ public void ensureLogGroupAndStream(String logGroup, String logStream, String region) {+ try {+ cloudWatchLogsService.createLogGroup(logGroup, null, null, region);+ } catch (AwsException ignored) {+ // Already exists+ } catch (Exception e) {+ LOG.warnv("Could not create CloudWatch log group {0}: {1}", logGroup, e.getMessage());+ }++ try {+ cloudWatchLogsService.createLogStream(logGroup, logStream, region);+ } catch (AwsException ignored) {+ // Already exists+ } catch (Exception e) {+ LOG.warnv("Could not create CloudWatch log stream {0}/{1}: {2}", logGroup, logStream, e.getMessage());+ }+ }++ /**+ * Generates a date-prefixed log stream name in the standard AWS format.+ *+ * @param suffix the suffix to append (e.g., "[$LATEST]abc123" or "containerId")+ * @return log stream name like "2024/01/15/suffix"+ */+ public String generateLogStreamName(String suffix) {+ return LOG_STREAM_DATE_FMT.format(LocalDate.now()) + "/" + suffix;+ }++ private void forwardToCloudWatchLogs(String logGroup, String logStream, String region, String line) {+ try {+ Map<String, Object> event = new HashMap<>();+ event.put("timestamp", System.currentTimeMillis());+ event.put("message", line);+ cloudWatchLogsService.putLogEvents(logGroup, logStream, List.of(event), region);+ } catch (Exception e) {+ LOG.debugv("Could not forward log line to CloudWatch Logs: {0}", e.getMessage());+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerSpec.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerSpec.javanew file mode 100644index 0000000000..1c351e2824--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerSpec.java@@ -0,0 +1,71 @@+package io.github.hectorvent.floci.core.common.docker;++import com.github.dockerjava.api.model.Bind;+import com.github.dockerjava.api.model.LogConfig;+import com.github.dockerjava.api.model.Mount;++import java.util.List;+import java.util.Map;++/**+ * Immutable specification for a Docker container to be created.+ * Use {@link ContainerBuilder} to construct instances of this record.+ *+ * @param image Docker image name (required)+ * @param name Container name (optional, Docker generates one if null)+ * @param env Environment variables as "KEY=value" strings+ * @param cmd Command to run (overrides image CMD)+ * @param entrypoint Entrypoint to use (overrides image ENTRYPOINT)+ * @param memoryBytes Memory limit in bytes (null = no limit)+ * @param portBindings Map of container port to host port (0 = dynamic allocation)+ * @param exposedPorts Ports to expose (required for port bindings)+ * @param networkMode Docker network name or mode (null = default bridge)+ * @param mounts Volume mounts (named volumes, bind mounts, tmpfs)+ * @param binds Legacy bind mounts (prefer mounts for new code)+ * @param extraHosts Extra /etc/hosts entries as "hostname:ip" strings+ * @param logConfig Docker log driver configuration (null = daemon default)+ */+public record ContainerSpec(+ String image,+ String name,+ List<String> env,+ List<String> cmd,+ List<String> entrypoint,+ Long memoryBytes,+ Map<Integer, Integer> portBindings,+ List<Integer> exposedPorts,+ String networkMode,+ List<Mount> mounts,+ List<Bind> binds,+ List<String> extraHosts,+ LogConfig logConfig+) {+ /**+ * Creates a minimal spec with just the image name.+ * All other fields will be null or empty lists.+ */+ public ContainerSpec(String image) {+ this(image, null, List.of(), null, null, null, Map.of(), List.of(), null, List.of(), List.of(), List.of(), null);+ }++ /**+ * Returns true if this spec has any port bindings configured.+ */+ public boolean hasPortBindings() {+ return portBindings != null && !portBindings.isEmpty();+ }++ /**+ * Returns true if this spec has a memory limit configured.+ */+ public boolean hasMemoryLimit() {+ return memoryBytes != null && memoryBytes > 0;+ }++ /**+ * Returns true if log rotation is configured.+ */+ public boolean hasLogConfig() {+ return logConfig != null;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/PortAllocator.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/PortAllocator.javanew file mode 100644index 0000000000..104890433b--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/PortAllocator.java@@ -0,0 +1,69 @@+package io.github.hectorvent.floci.core.common.docker;++import jakarta.enterprise.context.ApplicationScoped;+import org.jboss.logging.Logger;++import java.io.IOException;+import java.net.ServerSocket;++/**+ * Utility for allocating free TCP ports for Docker container port bindings.+ * Consolidates the free port discovery logic previously duplicated across+ * container managers (RDS, ElastiCache, MSK, ECR).+ */+@ApplicationScoped+public class PortAllocator {++ private static final Logger LOG = Logger.getLogger(PortAllocator.class);++ /**+ * Finds a free TCP port within the specified range.+ *+ * @param basePort the lowest port number to try (inclusive)+ * @param maxPort the highest port number to try (inclusive)+ * @return a free port within the range+ * @throws RuntimeException if no free port is available in the range+ */+ public int allocate(int basePort, int maxPort) {+ for (int port = basePort; port <= maxPort; port++) {+ if (isPortFree(port)) {+ LOG.debugv("Allocated port {0} from range {1}-{2}", port, basePort, maxPort);+ return port;+ }+ }+ throw new RuntimeException("No free port available in range " + basePort + "-" + maxPort);+ }++ /**+ * Finds any free TCP port using ephemeral port allocation.+ * This is the fastest method when any port will do.+ *+ * @return a free port+ * @throws RuntimeException if no free port can be allocated+ */+ public int allocateAny() {+ try (ServerSocket socket = new ServerSocket(0)) {+ socket.setReuseAddress(true);+ int port = socket.getLocalPort();+ LOG.debugv("Allocated ephemeral port {0}", port);+ return port;+ } catch (IOException e) {+ throw new RuntimeException("Could not find a free port", e);+ }+ }++ /**+ * Checks if a specific port is currently free.+ *+ * @param port the port to check+ * @return true if the port is available, false otherwise+ */+ public boolean isPortFree(int port) {+ try (ServerSocket socket = new ServerSocket(port)) {+ socket.setReuseAddress(true);+ return true;+ } catch (IOException e) {+ return false;+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.javaindex b18e0387a0..50e210b77a 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java@@ -1,34 +1,31 @@package io.github.hectorvent.floci.services.ecr.registry;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerDetector;-import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;+import io.github.hectorvent.floci.core.common.docker.PortAllocator;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.async.ResultCallback;-import com.github.dockerjava.api.command.CreateContainerResponse;import com.github.dockerjava.api.command.ExecCreateCmdResponse;-import com.github.dockerjava.api.command.InspectContainerResponse;import com.github.dockerjava.api.command.InspectExecResponse;-import com.github.dockerjava.api.exception.NotFoundException;-import com.github.dockerjava.api.model.Frame;-import com.github.dockerjava.api.model.Bind;import com.github.dockerjava.api.model.Container;-import com.github.dockerjava.api.model.ExposedPort;-import com.github.dockerjava.api.model.HostConfig;-import com.github.dockerjava.api.model.Mount;-import com.github.dockerjava.api.model.MountType;-import com.github.dockerjava.api.model.Ports;-import com.github.dockerjava.api.model.Volume;+import com.github.dockerjava.api.model.Frame;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.io.Closeable;import java.io.IOException;-import java.net.ServerSocket;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;import java.nio.file.Paths;+import java.util.ArrayList;import java.util.List;import java.util.concurrent.TimeUnit;@@ -46,29 +43,38 @@ public class EcrRegistryManager {private static final Logger LOG = Logger.getLogger(EcrRegistryManager.class);private static final int CONTAINER_INTERNAL_PORT = 5000;-private static final String NAMED_VOLUME = "floci-ecr-registry-data";- private final DockerClient dockerClient;- private final EmulatorConfig config;- private final ImageCacheService imageCacheService;+ private final ContainerBuilder containerBuilder;+ private final ContainerLifecycleManager lifecycleManager;+ private final ContainerLogStreamer logStreamer;private final ContainerDetector containerDetector;+ private final PortAllocator portAllocator;+ private final EmulatorConfig config;+ private final RegionResolver regionResolver;private volatile boolean started;private volatile boolean reconciled;private volatile int hostPort;private volatile String containerId;+ private volatile Closeable logStream;private volatile java.util.function.Consumer<List<String>> reconcileHook;@Inject- public EcrRegistryManager(DockerClient dockerClient,+ public EcrRegistryManager(ContainerBuilder containerBuilder,+ ContainerLifecycleManager lifecycleManager,+ ContainerLogStreamer logStreamer,+ ContainerDetector containerDetector,+ PortAllocator portAllocator,EmulatorConfig config,- ImageCacheService imageCacheService,- ContainerDetector containerDetector) {- this.dockerClient = dockerClient;- this.config = config;- this.imageCacheService = imageCacheService;+ RegionResolver regionResolver) {+ this.containerBuilder = containerBuilder;+ this.lifecycleManager = lifecycleManager;+ this.logStreamer = logStreamer;this.containerDetector = containerDetector;+ this.portAllocator = portAllocator;+ this.config = config;+ this.regionResolver = regionResolver;this.hostPort = config.services().ecr().registryBasePort();}@@ -122,93 +128,93 @@ public class EcrRegistryManager {}String name = config.services().ecr().registryContainerName();- Container existing = findExistingContainer(name);- if (existing != null) {- adoptExisting(existing);+ // Check for existing container to adopt+ var existing = lifecycleManager.findByName(name);+ if (existing.isPresent()) {+ adoptExisting(existing.get());runReconcileOnce();return;}- int chosenPort = allocatePort();+ // Allocate port+ int chosenPort = portAllocator.allocate(+ config.services().ecr().registryBasePort(),+ config.services().ecr().registryMaxPort());+String image = config.services().ecr().registryImage();- // Pull the registry image on demand. CI runners and fresh dev machines- // will not have it cached locally; without this we'd see- // "Status 404: No such image: registry:2" from createContainerCmd.- imageCacheService.ensureImageExists(image);+ // Build environment variables+ List<String> env = new ArrayList<>(List.of(+ "REGISTRY_STORAGE_DELETE_ENABLED=true",+ "REGISTRY_HTTP_ADDR=0.0.0.0:" + CONTAINER_INTERNAL_PORT+ ));++ // Build container spec+ ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image)+ .withName(name)+ .withEnv(env)+ .withPortBinding(CONTAINER_INTERNAL_PORT, chosenPort)+ .withDockerNetwork(config.services().ecr().dockerNetwork())+ .withLogRotation();++ // Handle persistence mounting based on storage configuration+ addPersistenceMounts(specBuilder, env);- ExposedPort exposed = ExposedPort.tcp(CONTAINER_INTERNAL_PORT);- Ports portBindings = new Ports();- portBindings.bind(exposed, Ports.Binding.bindPort(chosenPort));+ ContainerSpec spec = specBuilder.build();+ try {+ ContainerInfo info = lifecycleManager.createAndStart(spec);+ this.containerId = info.containerId();+ this.hostPort = chosenPort;+ this.started = true;+ LOG.infov("Started ECR backing registry {0} on host port {1}", name, chosenPort);++ // Attach log streaming (new feature)+ attachLogStream();+ } catch (Exception e) {+ throw new RuntimeException("Failed to start ECR backing registry container: " + e.getMessage(), e);+ }+ runReconcileOnce();+ }++ private void addPersistenceMounts(ContainerBuilder.Builder specBuilder, List<String> env) {String hostPersistentPath = config.storage().hostPersistentPath();boolean inContainer = containerDetector.isRunningInContainer();boolean isExplicitVolumeName = !hostPersistentPath.startsWith("/")&& !hostPersistentPath.startsWith(".");boolean isRelativeDefault = hostPersistentPath.startsWith(".");- HostConfig hostConfig = HostConfig.newHostConfig()- .withPortBindings(portBindings);-- List<String> env = new java.util.ArrayList<>(List.of(- "REGISTRY_STORAGE_DELETE_ENABLED=true",- "REGISTRY_HTTP_ADDR=0.0.0.0:" + CONTAINER_INTERNAL_PORT- ));-if (inContainer && isRelativeDefault) {- // Zero-config fallback for in-container Floci. A relative host path can't- // be bind-mounted into a sibling container (Docker Desktop rejects it).- // Use a dedicated named volume so `docker compose up` works without any- // FLOCI_STORAGE_HOST_PERSISTENT_PATH configuration.- hostConfig.withMounts(List.of(new Mount()- .withType(MountType.VOLUME)- .withSource(NAMED_VOLUME)- .withTarget("/var/lib/registry")));+ // Zero-config fallback for in-container Floci+ specBuilder.withNamedVolume(NAMED_VOLUME, "/var/lib/registry");LOG.infov("Floci in container with relative host-persistent-path ({0}); "+ "using named volume {1} for ECR registry data",hostPersistentPath, NAMED_VOLUME);} else if (isExplicitVolumeName) {- // User set hostPersistentPath to a Docker named-volume name — share the- // volume with the rest of Floci's persistent data and tell the registry- // to write inside it.+ // User set hostPersistentPath to a Docker named-volume nameString internalMountPath = "/app/data";- hostConfig.withBinds(new Bind(hostPersistentPath, new Volume(internalMountPath)));+ specBuilder.withBind(hostPersistentPath, internalMountPath);env.add("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=" + internalMountPath + "/ecr/registry");} else {- // Host path bind-mount. On host dev, hostPersistentPath matches- // persistentPath so the replace is a no-op and we bind the absolute- // container data path. In a container with an absolute host path set,- // replace rewrites the container-internal path to the host-side path.+ // Host path bind-mountString dataPath = Paths.get(config.services().ecr().dataPath(), "registry").toAbsolutePath().toString();String hostDataPath = dataPath.replace(config.storage().persistentPath(), hostPersistentPath);if (!inContainer) {ensureDataDir();}- hostConfig.withBinds(new Bind(hostDataPath, new Volume("/var/lib/registry")));+ specBuilder.withBind(hostDataPath, "/var/lib/registry");}+ }- config.services().ecr().dockerNetwork()- .or(() -> config.services().dockerNetwork())- .filter(n -> !n.isBlank())- .ifPresent(hostConfig::withNetworkMode);+ private void attachLogStream() {+ String shortId = containerId.length() >= 8 ? containerId.substring(0, 8) : containerId;+ String logGroup = "/aws/ecr/registry";+ String logStreamName = logStreamer.generateLogStreamName(shortId);+ String region = regionResolver.getDefaultRegion();- try {- CreateContainerResponse created = dockerClient.createContainerCmd(image)- .withName(name)- .withEnv(env)- .withExposedPorts(exposed)- .withHostConfig(hostConfig)- .exec();- this.containerId = created.getId();- dockerClient.startContainerCmd(containerId).exec();- this.hostPort = chosenPort;- this.started = true;- LOG.infov("Started ECR backing registry {0} on host port {1}", name, chosenPort);- } catch (Exception e) {- throw new RuntimeException("Failed to start ECR backing registry container: " + e.getMessage(), e);- }- runReconcileOnce();+ this.logStream = logStreamer.attach(+ containerId, logGroup, logStreamName, region, "ecr:registry");}private void runReconcileOnce() {@@ -218,8 +224,15 @@ public class EcrRegistryManager {try {// Give the registry a moment to be ready on first startfor (int i = 0; i < 10; i++) {- if (httpClient().ping()) break;- try { Thread.sleep(200); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); return; }+ if (httpClient().ping()) {+ break;+ }+ try {+ Thread.sleep(200);+ } catch (InterruptedException ie) {+ Thread.currentThread().interrupt();+ return;+ }}List<String> repos = httpClient().catalog();reconcileHook.accept(repos);@@ -249,6 +262,8 @@ public class EcrRegistryManager {long startMs = System.currentTimeMillis();StringBuilder output = new StringBuilder();+ DockerClient dockerClient = lifecycleManager.getDockerClient();+ExecCreateCmdResponse exec = dockerClient.execCreateCmd(containerId).withCmd("registry", "garbage-collect", "/etc/docker/registry/config.yml")@@ -299,77 +314,26 @@ public class EcrRegistryManager {LOG.infov("Leaving ECR backing registry container {0} running for next start-up", containerId);return;}- try {- dockerClient.stopContainerCmd(containerId).withTimeout(5).exec();- } catch (Exception e) {- LOG.warnv("Error stopping ECR registry container: {0}", e.getMessage());- }- try {- dockerClient.removeContainerCmd(containerId).withForce(true).exec();- } catch (Exception e) {- LOG.warnv("Error removing ECR registry container: {0}", e.getMessage());- }- }-- private Container findExistingContainer(String name) {- try {- List<Container> all = dockerClient.listContainersCmd().withShowAll(true).exec();- for (Container c : all) {- String[] names = c.getNames();- if (names == null) continue;- for (String n : names) {- if (n.equals("/" + name) || n.equals(name)) {- return c;- }- }- }- } catch (Exception e) {- LOG.debugv("Could not list containers while searching for {0}: {1}", name, e.getMessage());- }- return null;+ lifecycleManager.stopAndRemove(containerId, logStream);}private void adoptExisting(Container existing) {this.containerId = existing.getId();try {- InspectContainerResponse inspect = dockerClient.inspectContainerCmd(containerId).exec();- boolean running = Boolean.TRUE.equals(inspect.getState().getRunning());- if (!running) {- dockerClient.startContainerCmd(containerId).exec();- }- var bindings = inspect.getNetworkSettings().getPorts().getBindings();- var binding = bindings.get(ExposedPort.tcp(CONTAINER_INTERNAL_PORT));- if (binding != null && binding.length > 0) {- this.hostPort = Integer.parseInt(binding[0].getHostPortSpec());+ ContainerInfo info = lifecycleManager.adopt(containerId, List.of(CONTAINER_INTERNAL_PORT));+ var endpoint = info.getEndpoint(CONTAINER_INTERNAL_PORT);+ if (endpoint != null) {+ this.hostPort = endpoint.port();}this.started = true;LOG.infov("Adopted existing ECR registry container {0} on host port {1}",containerId, hostPort);- } catch (NotFoundException nf) {- this.containerId = null;++ // Attach log streaming to adopted container+ attachLogStream();} catch (Exception e) {LOG.warnv("Failed to adopt existing ECR registry container: {0}", e.getMessage());- }- }-- private int allocatePort() {- int base = config.services().ecr().registryBasePort();- int max = config.services().ecr().registryMaxPort();- for (int p = base; p <= max; p++) {- if (isPortFree(p)) {- return p;- }- }- throw new RuntimeException("No free port available in range "- + base + "-" + max + " for ECR registry");- }-- private boolean isPortFree(int port) {- try (ServerSocket s = new ServerSocket(port)) {- s.setReuseAddress(true);- return true;- } catch (IOException e) {- return false;+ this.containerId = null;}}diff --git a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.javaindex 66323abfce..1e2123c2ff 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecs/container/EcsContainerManager.java@@ -1,35 +1,28 @@package io.github.hectorvent.floci.services.ecs.container;import io.github.hectorvent.floci.config.EmulatorConfig;-import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerDetector;-import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsService;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.services.ecs.model.Container;import io.github.hectorvent.floci.services.ecs.model.ContainerDefinition;import io.github.hectorvent.floci.services.ecs.model.EcsTask;import io.github.hectorvent.floci.services.ecs.model.NetworkBinding;import io.github.hectorvent.floci.services.ecs.model.PortMapping;import io.github.hectorvent.floci.services.ecs.model.TaskDefinition;-import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;import com.github.dockerjava.api.DockerClient;-import com.github.dockerjava.api.async.ResultCallback;-import com.github.dockerjava.api.command.CreateContainerResponse;import com.github.dockerjava.api.model.ExposedPort;-import com.github.dockerjava.api.model.Frame;-import com.github.dockerjava.api.model.HostConfig;-import com.github.dockerjava.api.model.Ports;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.io.Closeable;-import java.nio.charset.StandardCharsets;import java.time.Instant;-import java.time.LocalDate;-import java.time.format.DateTimeFormatter;import java.util.ArrayList;-import java.util.HashMap;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;@@ -42,27 +35,26 @@ import java.util.Map;public class EcsContainerManager {private static final Logger LOG = Logger.getLogger(EcsContainerManager.class);- private static final DateTimeFormatter LOG_STREAM_DATE_FMT = DateTimeFormatter.ofPattern("yyyy/MM/dd");- private final DockerClient dockerClient;- private final ImageCacheService imageCacheService;+ private final ContainerBuilder containerBuilder;+ private final ContainerLifecycleManager lifecycleManager;+ private final ContainerLogStreamer logStreamer;private final ContainerDetector containerDetector;private final EmulatorConfig config;- private final CloudWatchLogsService cloudWatchLogsService;private final RegionResolver regionResolver;@Inject- public EcsContainerManager(DockerClient dockerClient,- ImageCacheService imageCacheService,+ public EcsContainerManager(ContainerBuilder containerBuilder,+ ContainerLifecycleManager lifecycleManager,+ ContainerLogStreamer logStreamer,ContainerDetector containerDetector,EmulatorConfig config,- CloudWatchLogsService cloudWatchLogsService,RegionResolver regionResolver) {- this.dockerClient = dockerClient;- this.imageCacheService = imageCacheService;+ this.containerBuilder = containerBuilder;+ this.lifecycleManager = lifecycleManager;+ this.logStreamer = logStreamer;this.containerDetector = containerDetector;this.config = config;- this.cloudWatchLogsService = cloudWatchLogsService;this.regionResolver = regionResolver;}@@ -78,47 +70,66 @@ public class EcsContainerManager {List<Container> runtimeContainers = new ArrayList<>();for (ContainerDefinition def : taskDef.getContainerDefinitions()) {- imageCacheService.ensureImageExists(def.getImage());-- List<ExposedPort> exposedPorts = buildExposedPorts(def);- HostConfig hostConfig = buildHostConfig(def, exposedPorts);-- applyDockerNetwork(hostConfig);-- List<String> envVars = buildEnvVars(def);String containerName = "floci-ecs-" + taskId + "-" + def.getName();- var createCmd = dockerClient.createContainerCmd(def.getImage())+ // Build container spec+ ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(def.getImage()).withName(containerName)- .withEnv(envVars)- .withHostConfig(hostConfig);+ .withEnv(buildEnvVars(def))+ .withDockerNetwork(config.services().ecs().dockerNetwork())+ .withLogRotation();++ // Add memory limit if specified+ if (def.getMemory() != null) {+ specBuilder.withMemoryMb(def.getMemory());+ }- if (!exposedPorts.isEmpty()) {- createCmd.withExposedPorts(exposedPorts);+ // Add port mappings. Publish to host only in native mode; in Docker+ // mode ECS consumers reach containers via the docker network IP.+ if (def.getPortMappings() != null) {+ boolean publishToHost = !containerDetector.isRunningInContainer();+ for (PortMapping pm : def.getPortMappings()) {+ if (publishToHost) {+ specBuilder.withDynamicPort(pm.containerPort());+ } else {+ specBuilder.withExposedPort(pm.containerPort());+ }+ }}++ // Add command and entrypoint if specifiedif (def.getCommand() != null && !def.getCommand().isEmpty()) {- createCmd.withCmd(def.getCommand());+ specBuilder.withCmd(def.getCommand());}if (def.getEntryPoint() != null && !def.getEntryPoint().isEmpty()) {- createCmd.withEntrypoint(def.getEntryPoint());+ specBuilder.withEntrypoint(def.getEntryPoint());}- CreateContainerResponse created = createCmd.exec();- String dockerId = created.getId();- LOG.infov("Created ECS container {0} for task {1} container {2}", dockerId, taskId, def.getName());+ ContainerSpec spec = specBuilder.build();- dockerClient.startContainerCmd(dockerId).exec();- LOG.infov("Started ECS container {0}", dockerId);+ // Create and start container+ ContainerInfo info = lifecycleManager.createAndStart(spec);+ String dockerId = info.containerId();++ LOG.infov("Created ECS container {0} for task {1} container {2}", dockerId, taskId, def.getName());+ // Resolve network bindings for ECS-specific modelList<NetworkBinding> networkBindings = resolveNetworkBindings(dockerId, def);+ // Build ECS container modelContainer container = buildContainer(task.getTaskArn(), def, dockerId, networkBindings, region);runtimeContainers.add(container);containerIds.put(def.getName(), dockerId);- Closeable logStream = attachLogStream(dockerId, def.getName(), taskDef.getFamily(), taskId, region);- if (logStream != null) {- logStreams.add(logStream);+ // Attach log streaming+ String logGroup = "/ecs/" + taskDef.getFamily();+ String logStream = logStreamer.generateLogStreamName(def.getName() + "/" + taskId);++ Closeable logHandle = logStreamer.attach(+ dockerId, logGroup, logStream, region,+ "ecs:" + taskDef.getFamily() + ":" + def.getName());+ if (logHandle != null) {+ logStreams.add(logHandle);}}@@ -138,61 +149,18 @@ public class EcsContainerManager {return;}+ // Close all log streams firstfor (Closeable logStream : handle.getLogStreams()) {- try { logStream.close(); } catch (Exception ignored) {}- }-- for (Map.Entry<String, String> entry : handle.getContainerIds().entrySet()) {- String dockerId = entry.getValue();- try {- dockerClient.stopContainerCmd(dockerId).withTimeout(5).exec();- } catch (Exception e) {- LOG.warnv("Error stopping ECS container {0}: {1}", dockerId, e.getMessage());- }try {- dockerClient.removeContainerCmd(dockerId).withForce(true).exec();- } catch (Exception e) {- LOG.warnv("Error removing ECS container {0}: {1}", dockerId, e.getMessage());+ logStream.close();+ } catch (Exception ignored) {}}- }- private List<ExposedPort> buildExposedPorts(ContainerDefinition def) {- List<ExposedPort> exposed = new ArrayList<>();- if (def.getPortMappings() != null) {- for (PortMapping pm : def.getPortMappings()) {- exposed.add(ExposedPort.tcp(pm.containerPort()));- }- }- return exposed;- }-- private HostConfig buildHostConfig(ContainerDefinition def, List<ExposedPort> exposedPorts) {- HostConfig hostConfig = HostConfig.newHostConfig();-- if (def.getMemory() != null) {- hostConfig.withMemory((long) def.getMemory() * 1024 * 1024);- }-- if (!containerDetector.isRunningInContainer() && !exposedPorts.isEmpty()) {- Ports portBindings = new Ports();- for (ExposedPort ep : exposedPorts) {- portBindings.bind(ep, Ports.Binding.bindPort(0)); // 0 = dynamic host port- }- hostConfig.withPortBindings(portBindings);+ // Stop and remove all containers+ for (Map.Entry<String, String> entry : handle.getContainerIds().entrySet()) {+ lifecycleManager.stopAndRemove(entry.getValue(), null);}-- return hostConfig;- }-- private void applyDockerNetwork(HostConfig hostConfig) {- config.services().ecs().dockerNetwork()- .or(() -> config.services().dockerNetwork())- .filter(n -> !n.isBlank())- .ifPresent(network -> {- hostConfig.withNetworkMode(network);- LOG.debugv("Attaching ECS container to network: {0}", network);- });}private List<String> buildEnvVars(ContainerDefinition def) {@@ -211,6 +179,7 @@ public class EcsContainerManager {return bindings;}+ DockerClient dockerClient = lifecycleManager.getDockerClient();var inspect = dockerClient.inspectContainerCmd(dockerId).exec();var portBindingsMap = inspect.getNetworkSettings().getPorts().getBindings();@@ -246,65 +215,11 @@ public class EcsContainerManager {return container;}- private Closeable attachLogStream(String dockerId, String containerName,- String family, String taskId, String region) {- String logGroup = "/ecs/" + family;- String logStream = LOG_STREAM_DATE_FMT.format(LocalDate.now()) + "/" + containerName + "/" + taskId;- ensureLogGroupAndStream(logGroup, logStream, region);-- try {- return dockerClient.logContainerCmd(dockerId)- .withStdOut(true)- .withStdErr(true)- .withFollowStream(true)- .withTimestamps(false)- .exec(new ResultCallback.Adapter<>() {- @Override- public void onNext(Frame frame) {- String line = new String(frame.getPayload(), StandardCharsets.UTF_8).stripTrailing();- if (!line.isEmpty()) {- LOG.infov("[ecs:{0}:{1}] {2}", family, containerName, line);- forwardToCloudWatchLogs(logGroup, logStream, region, line);- }- }- });- } catch (Exception e) {- LOG.warnv("Could not attach log stream for ECS container {0}: {1}", dockerId, e.getMessage());- return null;- }- }-- private void ensureLogGroupAndStream(String logGroup, String logStream, String region) {- try {- cloudWatchLogsService.createLogGroup(logGroup, null, null, region);- } catch (AwsException ignored) {- } catch (Exception e) {- LOG.warnv("Could not create CW log group {0}: {1}", logGroup, e.getMessage());- }- try {- cloudWatchLogsService.createLogStream(logGroup, logStream, region);- } catch (AwsException ignored) {- } catch (Exception e) {- LOG.warnv("Could not create CW log stream {0}/{1}: {2}", logGroup, logStream, e.getMessage());- }- }-- private void forwardToCloudWatchLogs(String logGroup, String logStream, String region, String line) {- try {- Map<String, Object> event = new HashMap<>();- event.put("timestamp", System.currentTimeMillis());- event.put("message", line);- cloudWatchLogsService.putLogEvents(logGroup, logStream, List.of(event), region);- } catch (Exception e) {- LOG.debugv("Could not forward ECS log line to CloudWatch Logs: {0}", e.getMessage());- }- }-private static String extractTaskId(String taskArn) {int slash = taskArn.lastIndexOf('/');return slash >= 0 ? taskArn.substring(slash + 1) : taskArn;}// Inner enum to avoid import cycle — mirrors model.TaskStatus for readability- private enum TaskStatus { RUNNING }+ private enum TaskStatus {RUNNING}}diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.javaindex 9f17525e7f..9ba9d08648 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java@@ -1,29 +1,20 @@package io.github.hectorvent.floci.services.elasticache.container;import io.github.hectorvent.floci.config.EmulatorConfig;-import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerDetector;-import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsService;-import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;-import com.github.dockerjava.api.DockerClient;-import com.github.dockerjava.api.async.ResultCallback;-import com.github.dockerjava.api.command.CreateContainerResponse;-import com.github.dockerjava.api.model.ExposedPort;-import com.github.dockerjava.api.model.Frame;-import com.github.dockerjava.api.model.HostConfig;-import com.github.dockerjava.api.model.Ports;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;-import java.io.IOException;-import java.net.ServerSocket;-import java.nio.charset.StandardCharsets;-import java.time.LocalDate;-import java.time.format.DateTimeFormatter;+import java.io.Closeable;import java.util.ArrayList;-import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.concurrent.ConcurrentHashMap;@@ -38,111 +29,76 @@ public class ElastiCacheContainerManager {private static final Logger LOG = Logger.getLogger(ElastiCacheContainerManager.class);private static final int BACKEND_PORT = 6379;- private static final DateTimeFormatter LOG_STREAM_DATE_FMT = DateTimeFormatter.ofPattern("yyyy/MM/dd");- private final DockerClient dockerClient;- private final ImageCacheService imageCacheService;+ private final ContainerBuilder containerBuilder;+ private final ContainerLifecycleManager lifecycleManager;+ private final ContainerLogStreamer logStreamer;private final ContainerDetector containerDetector;private final EmulatorConfig config;- private final CloudWatchLogsService cloudWatchLogsService;private final RegionResolver regionResolver;private final Map<String, ElastiCacheContainerHandle> activeContainers = new ConcurrentHashMap<>();@Inject- public ElastiCacheContainerManager(DockerClient dockerClient,- ImageCacheService imageCacheService,+ public ElastiCacheContainerManager(ContainerBuilder containerBuilder,+ ContainerLifecycleManager lifecycleManager,+ ContainerLogStreamer logStreamer,ContainerDetector containerDetector,EmulatorConfig config,- CloudWatchLogsService cloudWatchLogsService,RegionResolver regionResolver) {- this.dockerClient = dockerClient;- this.imageCacheService = imageCacheService;+ this.containerBuilder = containerBuilder;+ this.lifecycleManager = lifecycleManager;+ this.logStreamer = logStreamer;this.containerDetector = containerDetector;this.config = config;- this.cloudWatchLogsService = cloudWatchLogsService;this.regionResolver = regionResolver;}public ElastiCacheContainerHandle start(String groupId, String image) {LOG.infov("Starting ElastiCache backend container for group: {0}", groupId);- imageCacheService.ensureImageExists(image);- HostConfig hostConfig = buildHostConfig();String containerName = "floci-valkey-" + groupId;- config.services().elasticache().dockerNetwork()- .or(() -> config.services().dockerNetwork())- .ifPresent(network -> {- if (!network.isBlank()) {- hostConfig.withNetworkMode(network);- LOG.debugv("Attaching ElastiCache container to network: {0}", network);- }- });+ // Remove any stale container with the same name+ lifecycleManager.removeIfExists(containerName);- // Remove any stale container with the same name (from a previous interrupted run)- try {- dockerClient.removeContainerCmd(containerName).withForce(true).exec();- LOG.infov("Removed stale container {0} before creating new one", containerName);- } catch (com.github.dockerjava.api.exception.NotFoundException ignored) {- // No existing container — normal path- }-- CreateContainerResponse container = dockerClient.createContainerCmd(image)+ // Build container spec. Only publish the backend port to the host in+ // native mode — in Docker mode the JVM reaches the container via its+ // network IP, no host binding needed.+ ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image).withName(containerName)- .withEnv(List.of("VALKEY_EXTRA_FLAGS=--loglevel verbose"))- .withExposedPorts(ExposedPort.tcp(BACKEND_PORT))- .withHostConfig(hostConfig)- .exec();-- String containerId = container.getId();- LOG.infov("Created ElastiCache container {0} for group {1}", containerId, groupId);-- dockerClient.startContainerCmd(containerId).exec();- LOG.infov("Started ElastiCache container {0}", containerId);-- String backendHost;- int backendPort;+ .withEnv("VALKEY_EXTRA_FLAGS", "--loglevel verbose")+ .withDockerNetwork(config.services().elasticache().dockerNetwork())+ .withLogRotation();if (!containerDetector.isRunningInContainer()) {- // Retrieve the actual allocated host port from the container inspect- var inspect = dockerClient.inspectContainerCmd(containerId).exec();- var bindings = inspect.getNetworkSettings().getPorts().getBindings();- var binding = bindings.get(ExposedPort.tcp(BACKEND_PORT));- if (binding != null && binding.length > 0) {- backendPort = Integer.parseInt(binding[0].getHostPortSpec());- } else {- backendPort = BACKEND_PORT;- }- backendHost = "localhost";+ specBuilder.withDynamicPort(BACKEND_PORT);} else {- // Docker mode: use container IP on the docker network- var inspect = dockerClient.inspectContainerCmd(containerId).exec();- var networks = inspect.getNetworkSettings().getNetworks();- String containerIp = null;- if (networks != null) {- for (Map.Entry<String, ?> entry : networks.entrySet()) {- var netEntry = (com.github.dockerjava.api.model.ContainerNetwork) entry.getValue();- containerIp = netEntry.getIpAddress();- if (containerIp != null && !containerIp.isBlank()) {- break;- }- }- }- if (containerIp == null || containerIp.isBlank()) {- containerIp = inspect.getNetworkSettings().getIpAddress();- }- backendHost = containerIp;- backendPort = BACKEND_PORT;+ specBuilder.withExposedPort(BACKEND_PORT);}- LOG.infov("ElastiCache backend for group {0}: {1}:{2}", groupId, backendHost, backendPort);+ ContainerSpec spec = specBuilder.build();++ // Create and start container+ ContainerInfo info = lifecycleManager.createAndStart(spec);+ EndpointInfo endpoint = info.getEndpoint(BACKEND_PORT);++ LOG.infov("ElastiCache backend for group {0}: {1}", groupId, endpoint);ElastiCacheContainerHandle handle = new ElastiCacheContainerHandle(- containerId, groupId, backendHost, backendPort);+ info.containerId(), groupId, endpoint.host(), endpoint.port());activeContainers.put(groupId, handle);- String shortId = containerId.length() >= 8 ? containerId.substring(0, 8) : containerId;- attachLogStream(handle, groupId, containerId, shortId);+ // Attach log streaming+ String shortId = info.containerId().length() >= 8+ ? info.containerId().substring(0, 8)+ : info.containerId();+ String logGroup = "/aws/elasticache/cluster/" + groupId + "/engine-log";+ String logStream = logStreamer.generateLogStreamName(shortId);+ String region = regionResolver.getDefaultRegion();++ Closeable logHandle = logStreamer.attach(+ info.containerId(), logGroup, logStream, region, "elasticache:" + groupId);+ handle.setLogStream(logHandle);return handle;}@@ -152,25 +108,7 @@ public class ElastiCacheContainerManager {return;}activeContainers.remove(handle.getGroupId());- LOG.infov("Stopping ElastiCache container {0}", handle.getContainerId());-- if (handle.getLogStream() != null) {- try { handle.getLogStream().close(); } catch (Exception ignored) {}- }-- try {- dockerClient.stopContainerCmd(handle.getContainerId()).withTimeout(5).exec();- } catch (Exception e) {- LOG.warnv("Error stopping ElastiCache container {0}: {1}",- handle.getContainerId(), e.getMessage());- }-- try {- dockerClient.removeContainerCmd(handle.getContainerId()).withForce(true).exec();- } catch (Exception e) {- LOG.warnv("Error removing ElastiCache container {0}: {1}",- handle.getContainerId(), e.getMessage());- }+ lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());}public void stopAll() {@@ -182,82 +120,4 @@ public class ElastiCacheContainerManager {stop(handle);}}-- private HostConfig buildHostConfig() {- HostConfig hostConfig = HostConfig.newHostConfig();- if (!containerDetector.isRunningInContainer()) {- // Bind BACKEND_PORT → random host port so the JVM can reach the container- int freePort = findFreePort();- Ports portBindings = new Ports();- portBindings.bind(ExposedPort.tcp(BACKEND_PORT), Ports.Binding.bindPort(freePort));- hostConfig.withPortBindings(portBindings);- LOG.debugv("Native mode: binding container port 6379 → host port {0}", freePort);- }- return hostConfig;- }-- private static int findFreePort() {- try (ServerSocket s = new ServerSocket(0)) {- s.setReuseAddress(true);- return s.getLocalPort();- } catch (IOException e) {- throw new RuntimeException("Could not find a free port for ElastiCache container", e);- }- }-- private void attachLogStream(ElastiCacheContainerHandle handle, String groupId,- String containerId, String shortId) {- String logGroup = "/aws/elasticache/cluster/" + groupId + "/engine-log";- String region = regionResolver.getDefaultRegion();- String logStream = LOG_STREAM_DATE_FMT.format(LocalDate.now()) + "/" + shortId;- ensureLogGroupAndStream(logGroup, logStream, region);-- try {- ResultCallback.Adapter<Frame> logCallback = dockerClient.logContainerCmd(containerId)- .withStdOut(true)- .withStdErr(true)- .withFollowStream(true)- .withTimestamps(false)- .exec(new ResultCallback.Adapter<>() {- @Override- public void onNext(Frame frame) {- String line = new String(frame.getPayload(), StandardCharsets.UTF_8).stripTrailing();- if (!line.isEmpty()) {- LOG.infov("[elasticache:{0}] {1}", groupId, line);- forwardToCloudWatchLogs(logGroup, logStream, region, line);- }- }- });- handle.setLogStream(logCallback);- } catch (Exception e) {- LOG.warnv("Could not attach log stream for ElastiCache container {0}: {1}",- containerId, e.getMessage());- }- }-- private void ensureLogGroupAndStream(String logGroup, String logStream, String region) {- try {- cloudWatchLogsService.createLogGroup(logGroup, null, null, region);- } catch (AwsException ignored) {- } catch (Exception e) {- LOG.warnv("Could not create CW log group {0}: {1}", logGroup, e.getMessage());- }- try {- cloudWatchLogsService.createLogStream(logGroup, logStream, region);- } catch (AwsException ignored) {- } catch (Exception e) {- LOG.warnv("Could not create CW log stream {0}/{1}: {2}", logGroup, logStream, e.getMessage());- }- }-- private void forwardToCloudWatchLogs(String logGroup, String logStream, String region, String line) {- try {- Map<String, Object> event = new HashMap<>();- event.put("timestamp", System.currentTimeMillis());- event.put("message", line);- cloudWatchLogsService.putLogEvents(logGroup, logStream, List.of(event), region);- } catch (Exception e) {- LOG.debugv("Could not forward ElastiCache log line to CloudWatch Logs: {0}", e.getMessage());- }- }}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.javaindex 2df7d6b39d..ff887a33d6 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java@@ -1,20 +1,18 @@package io.github.hectorvent.floci.services.lambda.launcher;import io.github.hectorvent.floci.config.EmulatorConfig;-import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;-import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsService;import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;import io.github.hectorvent.floci.services.lambda.model.ContainerState;import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;import com.github.dockerjava.api.DockerClient;-import com.github.dockerjava.api.async.ResultCallback;-import com.github.dockerjava.api.command.CreateContainerCmd;-import com.github.dockerjava.api.command.CreateContainerResponse;-import com.github.dockerjava.api.model.Frame;-import com.github.dockerjava.api.model.HostConfig;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -22,17 +20,15 @@ import org.jboss.logging.Logger;import org.apache.commons.compress.archivers.tar.TarArchiveEntry;import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;+import java.io.Closeable;import java.io.IOException;import java.io.OutputStream;-import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;import java.time.LocalDate;import java.time.format.DateTimeFormatter;import java.util.ArrayList;-import java.util.HashMap;import java.util.List;-import java.util.Map;/*** Starts and stops Docker containers for Lambda function execution.@@ -51,13 +47,13 @@ public class ContainerLauncher {private static final DateTimeFormatter LOG_STREAM_DATE_FMT = DateTimeFormatter.ofPattern("yyyy/MM/dd");- private final DockerClient dockerClient;- private final ImageCacheService imageCacheService;+ private final ContainerBuilder containerBuilder;+ private final ContainerLifecycleManager lifecycleManager;+ private final ContainerLogStreamer logStreamer;private final ImageResolver imageResolver;private final RuntimeApiServerFactory runtimeApiServerFactory;private final DockerHostResolver dockerHostResolver;private final EmulatorConfig config;- private final CloudWatchLogsService cloudWatchLogsService;private final EcrRegistryManager ecrRegistryManager;/** Matches an AWS-shaped ECR image URI: {@code <account>.dkr.ecr.<region>.amazonaws.com/<repo>[:tag]}. */@@ -65,21 +61,21 @@ public class ContainerLauncher {java.util.regex.Pattern.compile("^([0-9]{12})\\.dkr\\.ecr\\.([a-z0-9-]+)\\.amazonaws\\.com/(.+)$");@Inject- public ContainerLauncher(DockerClient dockerClient,- ImageCacheService imageCacheService,- ImageResolver imageResolver,- RuntimeApiServerFactory runtimeApiServerFactory,- DockerHostResolver dockerHostResolver,- EmulatorConfig config,- CloudWatchLogsService cloudWatchLogsService,- EcrRegistryManager ecrRegistryManager) {- this.dockerClient = dockerClient;- this.imageCacheService = imageCacheService;+ public ContainerLauncher(ContainerBuilder containerBuilder,+ ContainerLifecycleManager lifecycleManager,+ ContainerLogStreamer logStreamer,+ ImageResolver imageResolver,+ RuntimeApiServerFactory runtimeApiServerFactory,+ DockerHostResolver dockerHostResolver,+ EmulatorConfig config,+ EcrRegistryManager ecrRegistryManager) {+ this.containerBuilder = containerBuilder;+ this.lifecycleManager = lifecycleManager;+ this.logStreamer = logStreamer;this.imageResolver = imageResolver;this.runtimeApiServerFactory = runtimeApiServerFactory;this.dockerHostResolver = dockerHostResolver;this.config = config;- this.cloudWatchLogsService = cloudWatchLogsService;this.ecrRegistryManager = ecrRegistryManager;}@@ -89,9 +85,13 @@ public class ContainerLauncher {* the rewrite is only applied immediately before the docker pull.*/private String rewriteForEmulatedRegistry(String image) {- if (image == null) return null;+ if (image == null) {+ return null;+ }java.util.regex.Matcher m = AWS_ECR_URI.matcher(image);- if (!m.matches()) return image;+ if (!m.matches()) {+ return image;+ }String account = m.group(1);String region = m.group(2);String repoAndTag = m.group(3);@@ -106,9 +106,6 @@ public class ContainerLauncher {LOG.infov("Launching container for function: {0}", fn.getFunctionName());// For Zip functions, verify code exists before allocating any resources.- // Without this check, a container could start with an empty /var/task if the- // function was deleted (or code was replaced) between the invocation being- // enqueued and the container being launched.if (fn.getCodeLocalPath() != null) {Path codePath = Path.of(fn.getCodeLocalPath());if (!Files.exists(codePath)) {@@ -127,12 +124,8 @@ public class ContainerLauncher {: imageResolver.resolve(fn.getRuntime());// If this is an AWS-shaped ECR URI, rewrite it to Floci's loopback registry- // so docker pull lands on the emulated backing store rather than real AWS.image = rewriteForEmulatedRegistry(image);- // Ensure image is available locally- imageCacheService.ensureImageExists(image);-// Determine host address reachable from containerString hostAddress = dockerHostResolver.resolve();String runtimeApiEndpoint = hostAddress + ":" + runtimeApiServer.getPort();@@ -153,103 +146,62 @@ public class ContainerLauncher {fn.getEnvironment().forEach((k, v) -> env.add(k + "=" + v));}- // Build host config — memory limit only, no bind mount- // (code is copied in via Docker API tar-copy after container creation)- HostConfig hostConfig = HostConfig.newHostConfig()- .withMemory(fn.getMemorySize() * 1024 * 1024L);-- // On native Linux Docker, host.docker.internal is NOT auto-injected into- // containers (unlike Docker Desktop on Mac/Windows). Add an extra-host- // mapping that resolves to the host gateway, so the in-container Lambda- // RIC can reach Floci's Runtime API server via the same hostname on every- // platform. Harmless on Docker Desktop (it just duplicates an existing- // entry). The "host-gateway" magic value is translated by Docker to the- // bridge gateway IP at container-create time.- if (dockerHostResolver.isLinuxHost()) {- hostConfig.withExtraHosts("host.docker.internal:host-gateway");- }-- // Attach to a specific Docker network if configured- config.services().lambda().dockerNetwork()- .or(() -> config.services().dockerNetwork())- .ifPresent(network -> {- if (!network.isBlank()) {- hostConfig.withNetworkMode(network);- LOG.debugv("Attaching Lambda container to network: {0}", network);- }- });-- // Give the container a human-readable name so it is identifiable in Docker Desktop+ // Give the container a human-readable nameString shortId = java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 8);String containerName = "floci-" + fn.getFunctionName() + "-" + shortId;- // Create container — CMD must be the handler name (Lambda entrypoint requires it as first arg)- // For Image package type without an explicit handler, omit CMD so the image's own CMD is used.- CreateContainerCmd createCmd = dockerClient.createContainerCmd(image)+ // Build container spec+ ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image).withName(containerName).withEnv(env)- .withHostConfig(hostConfig);+ .withMemoryMb(fn.getMemorySize())+ .withDockerNetwork(config.services().lambda().dockerNetwork())+ .withHostDockerInternalOnLinux()+ .withLogRotation();++ // For Image package type without an explicit handler, omit CMD so the image's own CMD is usedif (fn.getHandler() != null && !fn.getHandler().isBlank()) {- createCmd.withCmd(fn.getHandler());+ specBuilder.withCmd(fn.getHandler());}- CreateContainerResponse container = createCmd.exec();- String containerId = container.getId();+ ContainerSpec spec = specBuilder.build();++ // Create container (but don't start yet - need to copy code first for Zip packages)+ ContainerInfo info = lifecycleManager.createAndStart(spec);+ String containerId = info.containerId();LOG.infov("Created container {0} for function {1}", containerId, fn.getFunctionName());// Copy code into container via Docker API tar stream (works inside Docker too)+ DockerClient dockerClient = lifecycleManager.getDockerClient();if (fn.getCodeLocalPath() != null) {Path codePath = Path.of(fn.getCodeLocalPath());-+// 1. Always copy all code to /var/task (TASK_DIR)- copyDirToContainer(containerId, codePath, TASK_DIR, fn.getFunctionName());+ copyDirToContainer(dockerClient, containerId, codePath, TASK_DIR, fn.getFunctionName());// 2. For provided runtimes, also copy the 'bootstrap' file to /var/runtime (RUNTIME_DIR)- // matching real AWS Lambda behavior where /var/runtime/bootstrap is the entry point.if (isProvidedRuntime(fn.getRuntime())) {Path bootstrapPath = codePath.resolve("bootstrap");if (Files.exists(bootstrapPath)) {- copyFileToContainer(containerId, bootstrapPath, RUNTIME_DIR, "bootstrap", fn.getFunctionName());+ copyFileToContainer(dockerClient, containerId, bootstrapPath, RUNTIME_DIR, "bootstrap", fn.getFunctionName());} else {- LOG.warnv("Provided runtime function {0} is missing 'bootstrap' file in {1}",+ LOG.warnv("Provided runtime function {0} is missing 'bootstrap' file in {1}",fn.getFunctionName(), fn.getCodeLocalPath());}}}- // Start container- dockerClient.startContainerCmd(containerId).exec();- LOG.infov("Started container {0}", containerId);-ContainerHandle handle = new ContainerHandle(containerId, fn.getFunctionName(), runtimeApiServer, ContainerState.WARM);// Determine CloudWatch Logs destination for this container instanceString cwLogGroup = "/aws/lambda/" + fn.getFunctionName();String region = extractRegionFromArn(fn.getFunctionArn());String cwLogStream = LOG_STREAM_DATE_FMT.format(LocalDate.now()) + "/[$LATEST]" + shortId;- ensureLogGroupAndStream(cwLogGroup, cwLogStream, region);-- // Stream container stdout/stderr to the emulator logger AND to CloudWatch Logs- try {- ResultCallback.Adapter<Frame> logCallback = dockerClient.logContainerCmd(containerId)- .withStdOut(true)- .withStdErr(true)- .withFollowStream(true)- .withTimestamps(false)- .exec(new ResultCallback.Adapter<>() {- @Override- public void onNext(Frame frame) {- String line = new String(frame.getPayload(), StandardCharsets.UTF_8).stripTrailing();- if (!line.isEmpty()) {- LOG.infov("[lambda:{0}] {1}", fn.getFunctionName(), line);- forwardToCloudWatchLogs(cwLogGroup, cwLogStream, region, line);- }- }- });- handle.setLogStream(logCallback);- } catch (Exception e) {- LOG.warnv("Could not attach log stream for container {0}: {1}", containerId, e.getMessage());- }++ // Attach log streaming+ Closeable logHandle = logStreamer.attach(+ containerId, cwLogGroup, cwLogStream, region, "lambda:" + fn.getFunctionName());+ handle.setLogStream(logHandle);return handle;}@@ -258,55 +210,12 @@ public class ContainerLauncher {LOG.infov("Stopping container {0}", handle.getContainerId());handle.setState(ContainerState.STOPPED);- // Close log stream first so the streaming thread exits cleanly- if (handle.getLogStream() != null) {- try { handle.getLogStream().close(); } catch (Exception ignored) {}- }-- try {- dockerClient.stopContainerCmd(handle.getContainerId()).withTimeout(5).exec();- } catch (Exception e) {- LOG.warnv("Error stopping container {0}: {1}", handle.getContainerId(), e.getMessage());- }-- try {- dockerClient.removeContainerCmd(handle.getContainerId()).withForce(true).exec();- } catch (Exception e) {- LOG.warnv("Error removing container {0}: {1}", handle.getContainerId(), e.getMessage());- }-+ lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());handle.getRuntimeApiServer().stop();}- private void ensureLogGroupAndStream(String logGroup, String logStream, String region) {- try {- cloudWatchLogsService.createLogGroup(logGroup, null, null, region);- } catch (AwsException ignored) {- // Already exists — that's fine- } catch (Exception e) {- LOG.warnv("Could not create CW log group {0}: {1}", logGroup, e.getMessage());- }- try {- cloudWatchLogsService.createLogStream(logGroup, logStream, region);- } catch (AwsException ignored) {- // Already exists — that's fine- } catch (Exception e) {- LOG.warnv("Could not create CW log stream {0}/{1}: {2}", logGroup, logStream, e.getMessage());- }- }-- private void forwardToCloudWatchLogs(String logGroup, String logStream, String region, String line) {- try {- Map<String, Object> event = new HashMap<>();- event.put("timestamp", System.currentTimeMillis());- event.put("message", line);- cloudWatchLogsService.putLogEvents(logGroup, logStream, List.of(event), region);- } catch (Exception e) {- LOG.debugv("Could not forward log line to CloudWatch Logs: {0}", e.getMessage());- }- }-- private void copyDirToContainer(String containerId, Path sourceDir, String remotePath, String functionName) {+ private void copyDirToContainer(DockerClient dockerClient, String containerId,+ Path sourceDir, String remotePath, String functionName) {try (java.io.PipedOutputStream pos = new java.io.PipedOutputStream();java.io.PipedInputStream pis = new java.io.PipedInputStream(pos)) {@@ -328,7 +237,8 @@ public class ContainerLauncher {}}- private void copyFileToContainer(String containerId, Path sourceFile, String remotePath, String entryName, String functionName) {+ private void copyFileToContainer(DockerClient dockerClient, String containerId,+ Path sourceFile, String remotePath, String entryName, String functionName) {try (java.io.PipedOutputStream pos = new java.io.PipedOutputStream();java.io.PipedInputStream pis = new java.io.PipedInputStream(pos)) {diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java b/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.javaindex 28ae0db0a2..cfcdbecfac 100644--- a/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java@@ -1,30 +1,29 @@package io.github.hectorvent.floci.services.msk;-import com.github.dockerjava.api.DockerClient;-import com.github.dockerjava.api.command.CreateContainerResponse;-import com.github.dockerjava.api.model.Bind;-import com.github.dockerjava.api.model.ExposedPort;-import com.github.dockerjava.api.model.HostConfig;-import com.github.dockerjava.api.model.Ports;-import com.github.dockerjava.api.model.Volume;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerDetector;-import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.services.msk.model.MskCluster;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.io.Closeable;import java.io.IOException;import java.net.HttpURLConnection;-import java.net.ServerSocket;import java.net.URI;-import java.net.URL;import java.nio.file.Files;import java.nio.file.Path;-import java.time.Duration;+import java.util.ArrayList;import java.util.List;import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;@ApplicationScopedpublic class RedpandaManager {@@ -33,26 +32,32 @@ public class RedpandaManager {private static final int KAFKA_PORT = 9092;private static final int ADMIN_PORT = 9644;- private final DockerClient dockerClient;- private final ImageCacheService imageCacheService;+ private final ContainerBuilder containerBuilder;+ private final ContainerLifecycleManager lifecycleManager;+ private final ContainerLogStreamer logStreamer;private final ContainerDetector containerDetector;private final EmulatorConfig config;+ private final RegionResolver regionResolver;+ private final Map<String, Closeable> logStreams = new ConcurrentHashMap<>();@Inject- public RedpandaManager(DockerClient dockerClient,- ImageCacheService imageCacheService,+ public RedpandaManager(ContainerBuilder containerBuilder,+ ContainerLifecycleManager lifecycleManager,+ ContainerLogStreamer logStreamer,ContainerDetector containerDetector,- EmulatorConfig config) {- this.dockerClient = dockerClient;- this.imageCacheService = imageCacheService;+ EmulatorConfig config,+ RegionResolver regionResolver) {+ this.containerBuilder = containerBuilder;+ this.lifecycleManager = lifecycleManager;+ this.logStreamer = logStreamer;this.containerDetector = containerDetector;this.config = config;+ this.regionResolver = regionResolver;}public void startContainer(MskCluster cluster) {String image = config.services().msk().defaultImage();LOG.infov("Starting Redpanda container for MSK cluster: {0} using image {1}", cluster.getClusterName(), image);- imageCacheService.ensureImageExists(image);String containerName = "floci-msk-" + cluster.getClusterName();@@ -64,107 +69,86 @@ public class RedpandaManager {LOG.errorv("Failed to create MSK data directory: {0}", dataPath, e);}+ // Cleanup stale container+ lifecycleManager.removeIfExists(containerName);++ // Build command+ List<String> cmd = new ArrayList<>(List.of(+ "redpanda", "start", "--overprovisioned", "--smp", "1",+ "--memory", "512M", "--reserve-memory", "0M"));++ // Build container spec. Publish Kafka/admin ports to the host only in+ // native mode; in Docker mode producers/consumers reach the broker via+ // the docker network IP resolved from container inspect.+ ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image)+ .withName(containerName)+ .withDockerNetwork(config.services().dockerNetwork())+ .withLogRotation();++ if (!containerDetector.isRunningInContainer()) {+ specBuilder.withDynamicPort(KAFKA_PORT).withDynamicPort(ADMIN_PORT);+ } else {+ specBuilder.withExposedPort(KAFKA_PORT).withExposedPort(ADMIN_PORT);+ }++ // Handle persistence mountingString hostPersistentPath = config.storage().hostPersistentPath();boolean isVolume = !hostPersistentPath.startsWith("/") && !hostPersistentPath.startsWith(".");- HostConfig hostConfig = HostConfig.newHostConfig();- List<String> cmd = new java.util.ArrayList<>(List.of("redpanda", "start", "--overprovisioned", "--smp", "1", "--memory", "512M", "--reserve-memory", "0M"));-if (isVolume) {// Volume mode: mount the whole volume and point Redpanda to the subdirectoryString internalMountPath = "/app/data";- hostConfig.withBinds(new Bind(hostPersistentPath, new Volume(internalMountPath)));+ specBuilder.withBind(hostPersistentPath, internalMountPath);cmd.add("--data-dir");cmd.add(internalMountPath + "/msk/" + cluster.getClusterName());} else {// Directory mode: mount the specific subdirectory directlyString hostDataPath = Path.of(hostPersistentPath, "msk", cluster.getClusterName()).toAbsolutePath().toString();- hostConfig.withBinds(new Bind(hostDataPath, new Volume("/var/lib/redpanda/data")));+ specBuilder.withBind(hostDataPath, "/var/lib/redpanda/data");}- config.services().dockerNetwork()- .or(() -> config.services().dockerNetwork())- .ifPresent(network -> {- if (!network.isBlank()) {- hostConfig.withNetworkMode(network);- }- });+ specBuilder.withCmd(cmd);+ ContainerSpec spec = specBuilder.build();- Ports portBindings = new Ports();- int hostKafkaPort = KAFKA_PORT;- int hostAdminPort = ADMIN_PORT;-- if (!containerDetector.isRunningInContainer()) {- hostKafkaPort = findFreePort();- hostAdminPort = findFreePort();- portBindings.bind(ExposedPort.tcp(KAFKA_PORT), Ports.Binding.bindPort(hostKafkaPort));- portBindings.bind(ExposedPort.tcp(ADMIN_PORT), Ports.Binding.bindPort(hostAdminPort));- hostConfig.withPortBindings(portBindings);- }-- // Cleanup stale container- try {- dockerClient.removeContainerCmd(containerName).withForce(true).exec();- } catch (Exception ignored) {}+ // Create and start container+ ContainerInfo info = lifecycleManager.createAndStart(spec);+ cluster.setContainerId(info.containerId());- CreateContainerResponse container = dockerClient.createContainerCmd(image)- .withName(containerName)- .withExposedPorts(ExposedPort.tcp(KAFKA_PORT), ExposedPort.tcp(ADMIN_PORT))- .withHostConfig(hostConfig)- .withCmd(cmd)- .exec();+ // Resolve endpoints+ EndpointInfo kafkaEndpoint = info.getEndpoint(KAFKA_PORT);- String containerId = container.getId();- cluster.setContainerId(containerId);- dockerClient.startContainerCmd(containerId).exec();+ cluster.setBootstrapBrokers(kafkaEndpoint.host() + ":" + kafkaEndpoint.port());+ LOG.infov("Redpanda container {0} started. Bootstrap: {1}", info.containerId(), cluster.getBootstrapBrokers());- String bootstrapHost;- int bootstrapPort;- String adminHost;- int adminPort;+ // Attach log streaming (new feature)+ String shortId = info.containerId().length() >= 8+ ? info.containerId().substring(0, 8)+ : info.containerId();+ String logGroup = "/aws/msk/cluster/" + cluster.getClusterName();+ String logStream = logStreamer.generateLogStreamName(shortId);+ String region = regionResolver.getDefaultRegion();- if (!containerDetector.isRunningInContainer()) {- bootstrapHost = "localhost";- bootstrapPort = hostKafkaPort;- adminHost = "localhost";- adminPort = hostAdminPort;- } else {- var inspect = dockerClient.inspectContainerCmd(containerId).exec();- String containerIp = inspect.getNetworkSettings().getNetworks().values().stream()- .map(com.github.dockerjava.api.model.ContainerNetwork::getIpAddress)- .filter(ip -> ip != null && !ip.isBlank())- .findFirst()- .orElse(inspect.getNetworkSettings().getIpAddress());- bootstrapHost = containerIp;- bootstrapPort = KAFKA_PORT;- adminHost = containerIp;- adminPort = ADMIN_PORT;+ Closeable logHandle = logStreamer.attach(+ info.containerId(), logGroup, logStream, region, "msk:" + cluster.getClusterName());+ if (logHandle != null) {+ logStreams.put(cluster.getClusterName(), logHandle);}-- cluster.setBootstrapBrokers(bootstrapHost + ":" + bootstrapPort);- LOG.infov("Redpanda container {0} started. Bootstrap: {1}", containerId, cluster.getBootstrapBrokers());-- // Wait for readiness in a separate thread or return and let the service handle state?- // Service should handle the polling to CREATING -> ACTIVE transition.}public boolean isReady(MskCluster cluster) {String bootstrap = cluster.getBootstrapBrokers();- if (bootstrap == null) return false;-- // We need the admin API to check readiness properly as per design doc (/ready endpoint)- // For now, let's derive the admin URL.- // In native mode it's localhost:hostAdminPort. In container mode it's containerIp:9644.-+ if (bootstrap == null) {+ return false;+ }++ // Derive admin URL from the containerString adminUrl;if (!containerDetector.isRunningInContainer()) {- // We didn't store adminPort in cluster object, let's inspect again or store it.- // Let's store it in cluster for simplicity during development.- // Actually, let's just inspect the container to find the binding for 9644.+ var dockerClient = lifecycleManager.getDockerClient();var inspect = dockerClient.inspectContainerCmd(cluster.getContainerId()).exec();var bindings = inspect.getNetworkSettings().getPorts().getBindings();- var binding = bindings.get(ExposedPort.tcp(ADMIN_PORT));+ var binding = bindings.get(com.github.dockerjava.api.model.ExposedPort.tcp(ADMIN_PORT));if (binding != null && binding.length > 0) {adminUrl = "http://localhost:" + binding[0].getHostPortSpec() + "/ready";} else {@@ -187,22 +171,14 @@ public class RedpandaManager {}public void stopContainer(MskCluster cluster) {- if (cluster.getContainerId() == null) return;- try {- dockerClient.stopContainerCmd(cluster.getContainerId()).withTimeout(5).exec();- dockerClient.removeContainerCmd(cluster.getContainerId()).withForce(true).exec();- LOG.infov("Redpanda container {0} stopped and removed", cluster.getContainerId());- } catch (Exception e) {- LOG.warnv("Failed to stop Redpanda container {0}: {1}", cluster.getContainerId(), e.getMessage());+ if (cluster.getContainerId() == null) {+ return;}- }- private static int findFreePort() {- try (ServerSocket s = new ServerSocket(0)) {- s.setReuseAddress(true);- return s.getLocalPort();- } catch (IOException e) {- throw new RuntimeException("Could not find a free port for Redpanda", e);- }+ // Close log stream+ Closeable logHandle = logStreams.remove(cluster.getClusterName());++ lifecycleManager.stopAndRemove(cluster.getContainerId(), logHandle);+ LOG.infov("Redpanda container {0} stopped and removed", cluster.getContainerId());}}diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.javaindex dbf5bffc2f..5d5483d1bc 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java@@ -1,33 +1,24 @@package io.github.hectorvent.floci.services.rds.container;import io.github.hectorvent.floci.config.EmulatorConfig;-import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerDetector;-import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsService;-import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.services.rds.model.DatabaseEngine;-import com.github.dockerjava.api.DockerClient;-import com.github.dockerjava.api.async.ResultCallback;-import com.github.dockerjava.api.command.CreateContainerResponse;-import com.github.dockerjava.api.model.Bind;-import com.github.dockerjava.api.model.ExposedPort;-import com.github.dockerjava.api.model.Frame;-import com.github.dockerjava.api.model.HostConfig;-import com.github.dockerjava.api.model.Ports;-import com.github.dockerjava.api.model.Volume;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.io.Closeable;import java.io.IOException;-import java.net.ServerSocket;-import java.nio.charset.StandardCharsets;+import java.nio.file.Files;import java.nio.file.Path;-import java.time.LocalDate;-import java.time.format.DateTimeFormatter;import java.util.ArrayList;-import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.concurrent.ConcurrentHashMap;@@ -40,28 +31,27 @@ import java.util.concurrent.ConcurrentHashMap;public class RdsContainerManager {private static final Logger LOG = Logger.getLogger(RdsContainerManager.class);- private static final DateTimeFormatter LOG_STREAM_DATE_FMT = DateTimeFormatter.ofPattern("yyyy/MM/dd");- private final DockerClient dockerClient;- private final ImageCacheService imageCacheService;+ private final ContainerBuilder containerBuilder;+ private final ContainerLifecycleManager lifecycleManager;+ private final ContainerLogStreamer logStreamer;private final ContainerDetector containerDetector;private final EmulatorConfig config;- private final CloudWatchLogsService cloudWatchLogsService;private final RegionResolver regionResolver;private final Map<String, RdsContainerHandle> activeContainers = new ConcurrentHashMap<>();@Inject- public RdsContainerManager(DockerClient dockerClient,- ImageCacheService imageCacheService,+ public RdsContainerManager(ContainerBuilder containerBuilder,+ ContainerLifecycleManager lifecycleManager,+ ContainerLogStreamer logStreamer,ContainerDetector containerDetector,EmulatorConfig config,- CloudWatchLogsService cloudWatchLogsService,RegionResolver regionResolver) {- this.dockerClient = dockerClient;- this.imageCacheService = imageCacheService;+ this.containerBuilder = containerBuilder;+ this.lifecycleManager = lifecycleManager;+ this.logStreamer = logStreamer;this.containerDetector = containerDetector;this.config = config;- this.cloudWatchLogsService = cloudWatchLogsService;this.regionResolver = regionResolver;}@@ -69,118 +59,64 @@ public class RdsContainerManager {String image, String masterUsername,String masterPassword, String dbName) {LOG.infov("Starting RDS backend container for instance: {0} engine={1}", instanceId, engine);- imageCacheService.ensureImageExists(image);int enginePort = engine.defaultPort();+ String containerName = "floci-rds-" + instanceId;- String hostPersistentPath = config.storage().hostPersistentPath();- boolean isVolume = !hostPersistentPath.startsWith("/") && !hostPersistentPath.startsWith(".");+ // Remove any stale container with the same name+ lifecycleManager.removeIfExists(containerName);- HostConfig hostConfig = buildHostConfig(enginePort);+ // Build environment variablesList<String> envVars = buildEnvVars(engine, masterUsername, masterPassword, dbName);- if (isVolume) {- String internalMountPath = "/app/data";- hostConfig.withBinds(new Bind(hostPersistentPath, new Volume(internalMountPath)));- String dataPath = internalMountPath + "/rds/" + instanceId;- if (engine == DatabaseEngine.POSTGRES) {- envVars.add("PGDATA=" + dataPath);- } else {- // MySQL / MariaDB typically use /var/lib/mysql- // For volume mode, we can't easily remount a subpath to /var/lib/mysql- // but we can try to change the engine's data dir via CMD if needed.- // For now, let's use Bind for directory mode and warn for Volume mode if not Postgres.- LOG.warnv("Volume-based persistence for RDS engine {0} is currently only optimized for POSTGRES. Using default non-persistent path.", engine);- }+ // Build container spec with bind mounts for persistence. Publish the+ // engine port to the host only in native mode; in Docker mode the auth+ // proxy reaches the DB via the container network.+ ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image)+ .withName(containerName)+ .withEnv(envVars)+ .withDockerNetwork(config.services().rds().dockerNetwork())+ .withLogRotation();++ if (!containerDetector.isRunningInContainer()) {+ specBuilder.withDynamicPort(enginePort);} else {- String dataPath = Path.of(config.storage().persistentPath(), "rds", instanceId).toAbsolutePath().toString();- try {- java.nio.file.Files.createDirectories(Path.of(dataPath));- String hostDataPath = Path.of(hostPersistentPath, "rds", instanceId).toAbsolutePath().toString();- String target = (engine == DatabaseEngine.POSTGRES) ? "/var/lib/postgresql/data" : "/var/lib/mysql";- hostConfig.withBinds(new Bind(hostDataPath, new Volume(target)));- } catch (IOException e) {- LOG.errorv("Failed to create RDS data directory: {0}", dataPath, e);- }+ specBuilder.withExposedPort(enginePort);}- String containerName = "floci-rds-" + instanceId;-- config.services().rds().dockerNetwork()- .or(() -> config.services().dockerNetwork())- .ifPresent(network -> {- if (!network.isBlank()) {- hostConfig.withNetworkMode(network);- LOG.debugv("Attaching RDS container to network: {0}", network);- }- });-- // Remove any stale container with the same name (from a previous interrupted run)- try {- dockerClient.removeContainerCmd(containerName).withForce(true).exec();- LOG.infov("Removed stale container {0} before creating new one", containerName);- } catch (com.github.dockerjava.api.exception.NotFoundException ignored) {- // No existing container — normal path- }+ // Handle persistence mounting+ addPersistenceMounts(specBuilder, instanceId, engine, envVars);+ // Add engine-specific commandList<String> cmd = buildContainerCmd(engine);-- var createCmd = dockerClient.createContainerCmd(image)- .withName(containerName)- .withEnv(envVars)- .withExposedPorts(ExposedPort.tcp(enginePort))- .withHostConfig(hostConfig);-if (!cmd.isEmpty()) {- createCmd.withCmd(cmd);+ specBuilder.withCmd(cmd);}- CreateContainerResponse container = createCmd.exec();- String containerId = container.getId();- LOG.infov("Created RDS container {0} for instance {1}", containerId, instanceId);+ ContainerSpec spec = specBuilder.build();- dockerClient.startContainerCmd(containerId).exec();- LOG.infov("Started RDS container {0}", containerId);+ // Create and start container+ ContainerInfo info = lifecycleManager.createAndStart(spec);+ EndpointInfo endpoint = info.getEndpoint(enginePort);- String backendHost;- int backendPort;+ LOG.infov("RDS backend for instance {0}: {1}", instanceId, endpoint);- if (!containerDetector.isRunningInContainer()) {- var inspect = dockerClient.inspectContainerCmd(containerId).exec();- var bindings = inspect.getNetworkSettings().getPorts().getBindings();- var binding = bindings.get(ExposedPort.tcp(enginePort));- if (binding != null && binding.length > 0) {- backendPort = Integer.parseInt(binding[0].getHostPortSpec());- } else {- backendPort = enginePort;- }- backendHost = "localhost";- } else {- var inspect = dockerClient.inspectContainerCmd(containerId).exec();- var networks = inspect.getNetworkSettings().getNetworks();- String containerIp = null;- if (networks != null) {- for (Map.Entry<String, ?> entry : networks.entrySet()) {- var netEntry = (com.github.dockerjava.api.model.ContainerNetwork) entry.getValue();- containerIp = netEntry.getIpAddress();- if (containerIp != null && !containerIp.isBlank()) {- break;- }- }- }- if (containerIp == null || containerIp.isBlank()) {- containerIp = inspect.getNetworkSettings().getIpAddress();- }- backendHost = containerIp;- backendPort = enginePort;- }+ RdsContainerHandle handle = new RdsContainerHandle(+ info.containerId(), instanceId, endpoint.host(), endpoint.port());+ activeContainers.put(instanceId, handle);++ // Attach log streaming+ String shortId = info.containerId().length() >= 8+ ? info.containerId().substring(0, 8)+ : info.containerId();+ String logGroup = "/aws/rds/instance/" + instanceId + "/error";+ String logStream = logStreamer.generateLogStreamName(shortId);+ String region = regionResolver.getDefaultRegion();- LOG.infov("RDS backend for instance {0}: {1}:{2}", instanceId, backendHost, backendPort);+ Closeable logHandle = logStreamer.attach(+ info.containerId(), logGroup, logStream, region, "rds:" + instanceId);+ handle.setLogStream(logHandle);- RdsContainerHandle handle = new RdsContainerHandle(containerId, instanceId, backendHost, backendPort);- activeContainers.put(instanceId, handle);- String shortId = containerId.length() >= 8 ? containerId.substring(0, 8) : containerId;- attachLogStream(handle, instanceId, containerId, shortId);return handle;}@@ -189,23 +125,7 @@ public class RdsContainerManager {return;}activeContainers.remove(handle.getInstanceId());- LOG.infov("Stopping RDS container {0}", handle.getContainerId());-- if (handle.getLogStream() != null) {- try { handle.getLogStream().close(); } catch (Exception ignored) {}- }-- try {- dockerClient.stopContainerCmd(handle.getContainerId()).withTimeout(5).exec();- } catch (Exception e) {- LOG.warnv("Error stopping RDS container {0}: {1}", handle.getContainerId(), e.getMessage());- }-- try {- dockerClient.removeContainerCmd(handle.getContainerId()).withForce(true).exec();- } catch (Exception e) {- LOG.warnv("Error removing RDS container {0}: {1}", handle.getContainerId(), e.getMessage());- }+ lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());}public void stopAll() {@@ -218,50 +138,72 @@ public class RdsContainerManager {}}- private HostConfig buildHostConfig(int enginePort) {- HostConfig hostConfig = HostConfig.newHostConfig();- if (!containerDetector.isRunningInContainer()) {- int freePort = findFreePort();- Ports portBindings = new Ports();- portBindings.bind(ExposedPort.tcp(enginePort), Ports.Binding.bindPort(freePort));- hostConfig.withPortBindings(portBindings);- LOG.debugv("Native mode: binding container port {0} → host port {1}", enginePort, freePort);+ private void addPersistenceMounts(ContainerBuilder.Builder specBuilder, String instanceId,+ DatabaseEngine engine, List<String> envVars) {+ String hostPersistentPath = config.storage().hostPersistentPath();+ boolean isVolume = !hostPersistentPath.startsWith("/") && !hostPersistentPath.startsWith(".");++ if (isVolume) {+ // Named volume mode: mount the whole volume and tell the engine where to write+ String internalMountPath = "/app/data";+ specBuilder.withBind(hostPersistentPath, internalMountPath);++ String dataPath = internalMountPath + "/rds/" + instanceId;+ if (engine == DatabaseEngine.POSTGRES) {+ envVars.add("PGDATA=" + dataPath);+ } else {+ LOG.warnv("Volume-based persistence for RDS engine {0} is currently only optimized for POSTGRES. " ++ "Using default non-persistent path.", engine);+ }+ } else {+ // Directory mode: mount the specific subdirectory directly+ String dataPath = Path.of(config.storage().persistentPath(), "rds", instanceId)+ .toAbsolutePath().toString();+ try {+ Files.createDirectories(Path.of(dataPath));+ String hostDataPath = Path.of(hostPersistentPath, "rds", instanceId)+ .toAbsolutePath().toString();+ String target = (engine == DatabaseEngine.POSTGRES)+ ? "/var/lib/postgresql/data"+ : "/var/lib/mysql";+ specBuilder.withBind(hostDataPath, target);+ } catch (IOException e) {+ LOG.errorv("Failed to create RDS data directory: {0}", dataPath, e);+ }}- return hostConfig;}private List<String> buildEnvVars(DatabaseEngine engine, String masterUsername,String masterPassword, String dbName) {String effectiveUser = (masterUsername != null && !masterUsername.isBlank()) ? masterUsername : "postgres";- String effectiveDb = (dbName != null && !dbName.isBlank()) ? dbName : effectiveUser;- return switch (engine) {- case POSTGRES -> List.of(- "POSTGRES_USER=" + effectiveUser,- "POSTGRES_PASSWORD=" + masterPassword,- "POSTGRES_DB=" + effectiveDb,- "POSTGRES_HOST_AUTH_METHOD=md5"- );+ String effectiveDb = (dbName != null && !dbName.isBlank()) ? dbName : effectiveUser;++ List<String> envs = new ArrayList<>();+ switch (engine) {+ case POSTGRES -> {+ envs.add("POSTGRES_USER=" + effectiveUser);+ envs.add("POSTGRES_PASSWORD=" + masterPassword);+ envs.add("POSTGRES_DB=" + effectiveDb);+ envs.add("POSTGRES_HOST_AUTH_METHOD=md5");+ }case MYSQL -> {- var envs = new java.util.ArrayList<String>();envs.add("MYSQL_ROOT_PASSWORD=" + masterPassword);if (!"root".equals(effectiveUser)) {envs.add("MYSQL_USER=" + effectiveUser);envs.add("MYSQL_PASSWORD=" + masterPassword);}envs.add("MYSQL_DATABASE=" + effectiveDb);- yield envs;}case MARIADB -> {- var envs = new java.util.ArrayList<String>();envs.add("MARIADB_ROOT_PASSWORD=" + masterPassword);if (!"root".equals(effectiveUser)) {envs.add("MARIADB_USER=" + effectiveUser);envs.add("MARIADB_PASSWORD=" + masterPassword);}envs.add("MARIADB_DATABASE=" + effectiveDb);- yield envs;}- };+ }+ return envs;}private List<String> buildContainerCmd(DatabaseEngine engine) {@@ -272,69 +214,4 @@ public class RdsContainerManager {case POSTGRES, MARIADB -> List.of();};}-- private static int findFreePort() {- try (ServerSocket s = new ServerSocket(0)) {- s.setReuseAddress(true);- return s.getLocalPort();- } catch (IOException e) {- throw new RuntimeException("Could not find a free port for RDS container", e);- }- }-- private void attachLogStream(RdsContainerHandle handle, String instanceId,- String containerId, String shortId) {- String logGroup = "/aws/rds/instance/" + instanceId + "/error";- String region = regionResolver.getDefaultRegion();- String logStream = LOG_STREAM_DATE_FMT.format(LocalDate.now()) + "/" + shortId;- ensureLogGroupAndStream(logGroup, logStream, region);-- try {- ResultCallback.Adapter<Frame> logCallback = dockerClient.logContainerCmd(containerId)- .withStdOut(true)- .withStdErr(true)- .withFollowStream(true)- .withTimestamps(false)- .exec(new ResultCallback.Adapter<>() {- @Override- public void onNext(Frame frame) {- String line = new String(frame.getPayload(), StandardCharsets.UTF_8).stripTrailing();- if (!line.isEmpty()) {- LOG.infov("[rds:{0}] {1}", instanceId, line);- forwardToCloudWatchLogs(logGroup, logStream, region, line);- }- }- });- handle.setLogStream(logCallback);- } catch (Exception e) {- LOG.warnv("Could not attach log stream for RDS container {0}: {1}",- containerId, e.getMessage());- }- }-- private void ensureLogGroupAndStream(String logGroup, String logStream, String region) {- try {- cloudWatchLogsService.createLogGroup(logGroup, null, null, region);- } catch (AwsException ignored) {- } catch (Exception e) {- LOG.warnv("Could not create CW log group {0}: {1}", logGroup, e.getMessage());- }- try {- cloudWatchLogsService.createLogStream(logGroup, logStream, region);- } catch (AwsException ignored) {- } catch (Exception e) {- LOG.warnv("Could not create CW log stream {0}/{1}: {2}", logGroup, logStream, e.getMessage());- }- }-- private void forwardToCloudWatchLogs(String logGroup, String logStream, String region, String line) {- try {- Map<String, Object> event = new HashMap<>();- event.put("timestamp", System.currentTimeMillis());- event.put("message", line);- cloudWatchLogsService.putLogEvents(logGroup, logStream, List.of(event), region);- } catch (Exception e) {- LOG.debugv("Could not forward RDS log line to CloudWatch Logs: {0}", e.getMessage());- }- }}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 1a47e95812..23faf246ce 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -54,6 +54,10 @@ floci:validate-signatures: falsepresign-secret: local-emulator-secret+ docker:+ log-max-size: "10m"+ log-max-file: "3"+services:ssm:enabled: truediff --git a/src/test/java/io/github/hectorvent/floci/services/eventbridge/EventBridgeSchedulerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/eventbridge/EventBridgeSchedulerIntegrationTest.javaindex 83e95553a8..f572a48dcf 100644--- a/src/test/java/io/github/hectorvent/floci/services/eventbridge/EventBridgeSchedulerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/eventbridge/EventBridgeSchedulerIntegrationTest.java@@ -296,6 +296,8 @@ class EventBridgeSchedulerIntegrationTest {@Overridepublic ServicesConfig services() { return null; }@Override+ public DockerConfig docker() { return null; }+ @Overridepublic EmulatorConfig.InitHooksConfig initHooks() { return null; }};}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 88a7483cf9..2edb9ddf55 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -43,6 +43,10 @@ floci:timeout-seconds: 5shutdown-grace-period-seconds: 2+ docker:+ log-max-size: "10m"+ log-max-file: "3"+services:ssm:enabled: true