Light, fluffy, and always free - The AWS Local Emulator alternative
feat: implement Amazon MSK service with Redpanda orchestration (#419)
Hector Ventura91071aaae70f
diff --git a/Dockerfile b/Dockerfileindex 2082e259e3..4073122b82 100644--- a/Dockerfile+++ b/Dockerfile@@ -26,5 +26,6 @@ HEALTHCHECK --interval=5s --timeout=3s --retries=5 \ARG VERSION=latestENV FLOCI_VERSION=${VERSION}+ENV FLOCI_STORAGE_PERSISTENT_PATH=/app/dataENTRYPOINT ["java", "-jar", "quarkus-app/quarkus-run.jar"]diff --git a/Dockerfile.jvm-package b/Dockerfile.jvm-packageindex 04ed8ac290..4cca493781 100644--- a/Dockerfile.jvm-package+++ b/Dockerfile.jvm-package@@ -4,6 +4,7 @@ ARG VERSION=latestARG INSTALL_AWS_CLI=falseENV FLOCI_VERSION=${VERSION}+ENV FLOCI_STORAGE_PERSISTENT_PATH=/app/dataWORKDIR /appdiff --git a/Dockerfile.native b/Dockerfile.nativeindex e6d52820d7..815c105154 100644--- a/Dockerfile.native+++ b/Dockerfile.native@@ -30,6 +30,7 @@ HEALTHCHECK --interval=5s --timeout=3s --retries=5 \ARG VERSION=latestENV FLOCI_VERSION=${VERSION}+ENV FLOCI_STORAGE_PERSISTENT_PATH=/app/dataCOPY --from=build /app/target/*-runner /app/applicationRUN chmod +x /app/applicationdiff --git a/Dockerfile.native-package b/Dockerfile.native-packageindex 6f542cc7b3..10b5831734 100644--- a/Dockerfile.native-package+++ b/Dockerfile.native-package@@ -4,6 +4,7 @@ ARG VERSION=latestARG BINARY_FILE=applicationENV FLOCI_VERSION=${VERSION}+ENV FLOCI_STORAGE_PERSISTENT_PATH=/app/dataWORKDIR /appdiff --git a/README.md b/README.mdindex b4b26fb67a..2d4d1910cc 100644--- a/README.md+++ b/README.md@@ -44,6 +44,7 @@| Cognito | ✅ | ❌ || ElastiCache (Redis + IAM auth) | ✅ | ❌ || RDS (PostgreSQL + MySQL + IAM auth) | ✅ | ❌ |+| MSK (Kafka + Redpanda) | ✅ | ❌ || S3 Object Lock (COMPLIANCE / GOVERNANCE) | ✅ | ⚠️ Partial || DynamoDB Streams | ✅ | ⚠️ Partial || IAM (users, roles, policies, groups) | ✅ | ⚠️ Partial |@@ -74,7 +75,7 @@ flowchart LRendsubgraph Containers ["Container Services 🐳"]- C["Lambda\nElastiCache\nRDS\nECS"]+ C["Lambda\nElastiCache\nRDS\nECS\nMSK"]endRouter --> Stateless@@ -115,6 +116,7 @@ flowchart LR| **CloudWatch Metrics** | In-process | Custom metrics, statistics, alarms || **ElastiCache** | **Real Docker containers** | Redis / Valkey, IAM auth, SigV4 validation || **RDS** | **Real Docker containers** | PostgreSQL & MySQL, IAM auth, JDBC-compatible |+| **MSK** | **Real Docker containers** | Kafka compatible via Redpanda orchestration || **ECS** | **Real Docker containers** | Clusters, task definitions, tasks, services, capacity providers, task sets || **EC2** | In-process | VPCs, subnets, security groups, instances, AMIs, key pairs, internet gateways, route tables, Elastic IPs, tags || **ACM** | In-process | Certificate issuance, validation lifecycle |@@ -125,11 +127,11 @@ flowchart LR| **AppConfig** | In-process | Applications, environments, profiles, hosted configuration versions, deployments || **AppConfigData** | In-process | Configuration sessions, dynamic configuration retrieval |-> **Lambda, ElastiCache, RDS, and ECS** spin up real Docker containers and support IAM authentication and SigV4 request signing — the same auth flow as production AWS. **ECR** runs a shared `registry:2` container so the stock `docker` client can push and pull image bytes against repositories returned by the AWS-shaped control plane.+> **Lambda, ElastiCache, RDS, MSK, and ECS** spin up real Docker containers and support IAM authentication and SigV4 request signing — the same auth flow as production AWS. **ECR** runs a shared `registry:2` container so the stock `docker` client can push and pull image bytes against repositories returned by the AWS-shaped control plane.>> For per-service operation counts and endpoint protocols, see the [Services Overview](https://floci.io/floci/services/) in the documentation site.-**29 AWS services supported.**+**30 AWS services supported.**## Quick Startdiff --git a/compatibility-tests/sdk-test-java/pom.xml b/compatibility-tests/sdk-test-java/pom.xmlindex 394c73c77d..30bd8d9416 100644--- a/compatibility-tests/sdk-test-java/pom.xml+++ b/compatibility-tests/sdk-test-java/pom.xml@@ -106,6 +106,10 @@<groupId>software.amazon.awssdk</groupId><artifactId>iam</artifactId></dependency>+ <dependency>+ <groupId>software.amazon.awssdk</groupId>+ <artifactId>kafka</artifactId>+ </dependency><dependency><groupId>software.amazon.awssdk</groupId><artifactId>ec2</artifactId>diff --git a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.javaindex 8e943ea4d1..25ad115741 100644--- a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java+++ b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java@@ -26,6 +26,7 @@ import software.amazon.awssdk.services.sns.SnsClient;import software.amazon.awssdk.services.sqs.SqsClient;import software.amazon.awssdk.services.ssm.SsmClient;import software.amazon.awssdk.services.sts.StsClient;+import software.amazon.awssdk.services.kafka.KafkaClient;import software.amazon.awssdk.services.apigateway.ApiGatewayClient;import software.amazon.awssdk.services.apigatewayv2.ApiGatewayV2Client;import software.amazon.awssdk.services.elasticache.ElastiCacheClient;@@ -270,6 +271,14 @@ public final class TestFixtures {.build();}+ public static KafkaClient kafkaClient() {+ return KafkaClient.builder()+ .endpointOverride(ENDPOINT)+ .region(REGION)+ .credentialsProvider(CREDENTIALS)+ .build();+ }+public static KmsClient kmsClient() {return KmsClient.builder().endpointOverride(ENDPOINT)diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/MskTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/MskTest.javanew file mode 100644index 0000000000..e0c6fd90f9--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/MskTest.java@@ -0,0 +1,97 @@+package com.floci.test;++import org.junit.jupiter.api.*;+import software.amazon.awssdk.services.kafka.KafkaClient;+import software.amazon.awssdk.services.kafka.model.*;++import java.util.List;++import static org.assertj.core.api.Assertions.assertThat;++@DisplayName("MSK (Managed Streaming for Kafka)")+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class MskTest {++ private static KafkaClient kafka;+ private static String clusterArn;+ private static final String CLUSTER_NAME = TestFixtures.uniqueName("msk-cluster");++ @BeforeAll+ static void setup() {+ kafka = TestFixtures.kafkaClient();+ }++ @AfterAll+ static void cleanup() {+ if (kafka != null) {+ if (clusterArn != null) {+ try {+ kafka.deleteCluster(DeleteClusterRequest.builder().clusterArn(clusterArn).build());+ } catch (Exception ignored) {}+ }+ kafka.close();+ }+ }++ @Test+ @Order(1)+ void createCluster() {+ CreateClusterResponse response = kafka.createCluster(CreateClusterRequest.builder()+ .clusterName(CLUSTER_NAME)+ .kafkaVersion("3.6.1")+ .numberOfBrokerNodes(1)+ .brokerNodeGroupInfo(BrokerNodeGroupInfo.builder()+ .instanceType("kafka.m5.large")+ .clientSubnets("subnet-12345")+ .build())+ .build());++ assertThat(response.clusterArn()).isNotNull();+ assertThat(response.clusterName()).isEqualTo(CLUSTER_NAME);+ assertThat(response.state()).isIn(ClusterState.CREATING, ClusterState.ACTIVE);+ clusterArn = response.clusterArn();+ }++ @Test+ @Order(2)+ void describeCluster() {+ DescribeClusterResponse response = kafka.describeCluster(DescribeClusterRequest.builder()+ .clusterArn(clusterArn)+ .build());++ assertThat(response.clusterInfo()).isNotNull();+ assertThat(response.clusterInfo().clusterArn()).isEqualTo(clusterArn);+ assertThat(response.clusterInfo().clusterName()).isEqualTo(CLUSTER_NAME);+ }++ @Test+ @Order(3)+ void listClusters() {+ ListClustersResponse response = kafka.listClusters(ListClustersRequest.builder().build());++ assertThat(response.clusterInfoList()).anyMatch(c -> c.clusterArn().equals(clusterArn));+ }++ @Test+ @Order(4)+ void getBootstrapBrokers() {+ GetBootstrapBrokersResponse response = kafka.getBootstrapBrokers(GetBootstrapBrokersRequest.builder()+ .clusterArn(clusterArn)+ .build());++ // In mock mode it's immediate, in real mode it might be null while CREATING+ // but our MskService handles mock=true by setting it ACTIVE immediately.+ assertThat(response.bootstrapBrokerString()).isNotNull();+ }++ @Test+ @Order(5)+ void deleteCluster() {+ DeleteClusterResponse response = kafka.deleteCluster(DeleteClusterRequest.builder()+ .clusterArn(clusterArn)+ .build());++ assertThat(response.clusterArn()).isEqualTo(clusterArn);+ assertThat(response.state()).isEqualTo(ClusterState.DELETING);+ }+}diff --git a/docker-compose.yml b/docker-compose.ymlindex 0b936a43ad..63e112e0e1 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -12,10 +12,11 @@ services:- /var/run/docker.sock:/var/run/docker.sock- ./scripts:/etc/floci/init/start.d- ./scripts:/etc/floci/init/stop.d+ - ./data:/app/dataenvironment:FLOCI_SERVICES_DOCKER_NETWORK: floci_defaultFLOCI_SERVICES_RDS_PROXY_BASE_PORT: "7001"- FLOCI_STORAGE_PERSISTENT_PATH: /app/data+ FLOCI_STORAGE_HOST_PERSISTENT_PATH: ${PWD}/datanetworks:floci_default:aliases:diff --git a/docs/services/msk.md b/docs/services/msk.mdnew file mode 100644index 0000000000..69ffffc501--- /dev/null+++ b/docs/services/msk.md@@ -0,0 +1,62 @@+# MSK (Managed Streaming for Kafka)++**Protocol:** REST-JSON+**Endpoint:** `http://localhost:4566/`++Floci emulates Amazon MSK by orchestrating **Redpanda** containers. This provides high compatibility with the Kafka API while maintaining a low footprint.++## Supported Actions++| Action | Description |+|---|---|+| `CreateCluster` | Spawns a new Redpanda container for the cluster |+| `CreateClusterV2` | Modern serverless/provisioned creation (mapped to provisioned) |+| `ListClusters` | List all emulated clusters |+| `ListClustersV2` | List all emulated clusters using V2 API |+| `DescribeCluster` | Get cluster metadata and state |+| `DescribeClusterV2` | Get cluster metadata and state using V2 API |+| `DeleteCluster` | Stops and removes the Redpanda container |+| `GetBootstrapBrokers` | Get the connection strings for the cluster |++## Configuration++```yaml+floci:+ services:+ msk:+ enabled: true+ mock: false # Set to true for metadata-only CRUD (no Docker)+ default-image: "redpandadata/redpanda:latest"+```++## How it works++When `mock` is set to `false` (default), Floci uses the Docker API to start a Redpanda container for each created cluster.++- **Port Mapping**: The Kafka API (9092) is mapped to a dynamic host port.+- **Persistence**: Data is stored in the Floci persistent path under `msk/<cluster-name>`.+- **Readiness**: The cluster state transitions to `ACTIVE` once the Redpanda `/ready` endpoint is reachable.++## Examples++```bash+export AWS_ENDPOINT_URL=http://localhost:4566++# Create a cluster+aws kafka create-cluster \+ --cluster-name my-cluster \+ --kafka-version "3.6.1" \+ --numberOfBrokerNodes 1 \+ --broker-node-group-info '{"InstanceType":"kafka.m5.large","ClientSubnets":["subnet-1"]}' \+ --endpoint-url $AWS_ENDPOINT_URL++# List clusters+aws kafka list-clusters --endpoint-url $AWS_ENDPOINT_URL++# Get bootstrap brokers+CLUSTER_ARN=$(aws kafka list-clusters --query 'ClusterInfoList[0].ClusterArn' --output text --endpoint-url $AWS_ENDPOINT_URL)+aws kafka get-bootstrap-brokers --cluster-arn $CLUSTER_ARN --endpoint-url $AWS_ENDPOINT_URL++# Delete a cluster+aws kafka delete-cluster --cluster-arn $CLUSTER_ARN --endpoint-url $AWS_ENDPOINT_URL+```diff --git a/mkdocs.yml b/mkdocs.ymlindex f76d782f0a..f71ae8870d 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -85,6 +85,7 @@ nav:- STS: services/sts.md- ElastiCache: services/elasticache.md- RDS: services/rds.md+ - MSK (Kafka): services/msk.md- EventBridge: services/eventbridge.md- EventBridge Scheduler: services/scheduler.md- CloudWatch: services/cloudwatch.mddiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 22ab2e6027..ee8e080f6a 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -62,6 +62,10 @@ public interface EmulatorConfig {@WithDefault("./data")String persistentPath();+ /** The path on the host machine where data is stored. Useful for Docker-in-Docker. */+ @WithDefault("${floci.storage.persistent-path}")+ String hostPersistentPath();+WalConfig wal();ServiceStorageOverrides services();@@ -202,6 +206,7 @@ public interface EmulatorConfig {LambdaServiceConfig lambda();ApiGatewayServiceConfig apigateway();IamServiceConfig iam();+ MskServiceConfig msk();ElastiCacheServiceConfig elasticache();RdsServiceConfig rds();EventBridgeServiceConfig eventbridge();@@ -275,6 +280,17 @@ public interface EmulatorConfig {boolean enforcementEnabled();}+ interface MskServiceConfig {+ @WithDefault("true")+ boolean enabled();++ @WithDefault("false")+ boolean mock();++ @WithDefault("redpandadata/redpanda:latest")+ String defaultImage();+ }+interface ElastiCacheServiceConfig {@WithDefault("true")boolean enabled();@@ -442,7 +458,7 @@ public interface EmulatorConfig {@WithDefault("5199")int registryMaxPort();- @WithDefault("./data/ecr")+ @WithDefault("${floci.storage.persistent-path}/ecr")String dataPath();@WithDefault("false")diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.javaindex 299ade2917..24f8a70b35 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java@@ -64,6 +64,10 @@ public class ResolvedServiceCatalog {"iam", config.storage().mode(), 5000L, AwsNamespaces.IAM, ServiceProtocol.QUERY,protocols(ServiceProtocol.QUERY),Set.of(), Set.of("iam"), Set.of(), Set.of()),+ descriptor("kafka", "msk", config.services().msk().enabled(), true,+ "msk", config.storage().mode(), 5000L, null, ServiceProtocol.REST_JSON,+ protocols(ServiceProtocol.REST_JSON),+ Set.of(), Set.of("kafka"), Set.of(), Set.of(io.github.hectorvent.floci.services.msk.MskController.class)),descriptor("sts", "iam", config.services().iam().enabled(), false,null, null, 5000L, AwsNamespaces.STS, ServiceProtocol.QUERY,protocols(ServiceProtocol.QUERY),diff --git a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.javaindex 04272608ea..5cc9f4ca80 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java@@ -134,10 +134,26 @@ public class EcrRegistryManager {Ports portBindings = new Ports();portBindings.bind(exposed, Ports.Binding.bindPort(chosenPort));- String dataPath = Paths.get(config.services().ecr().dataPath(), "registry").toAbsolutePath().toString();+ String hostPersistentPath = config.storage().hostPersistentPath();+ boolean isVolume = !hostPersistentPath.startsWith("/") && !hostPersistentPath.startsWith(".");+HostConfig hostConfig = HostConfig.newHostConfig()- .withPortBindings(portBindings)- .withBinds(new Bind(dataPath, new Volume("/var/lib/registry")));+ .withPortBindings(portBindings);++ List<String> env = new java.util.ArrayList<>(List.of(+ "REGISTRY_STORAGE_DELETE_ENABLED=true",+ "REGISTRY_HTTP_ADDR=0.0.0.0:" + CONTAINER_INTERNAL_PORT+ ));++ if (isVolume) {+ String internalMountPath = "/app/data";+ hostConfig.withBinds(new Bind(hostPersistentPath, new Volume(internalMountPath)));+ env.add("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=" + internalMountPath + "/ecr/registry");+ } else {+ String dataPath = Paths.get(config.services().ecr().dataPath(), "registry").toAbsolutePath().toString();+ String hostDataPath = dataPath.replace(config.storage().persistentPath(), hostPersistentPath);+ hostConfig.withBinds(new Bind(hostDataPath, new Volume("/var/lib/registry")));+ }config.services().ecr().dockerNetwork().or(() -> config.services().dockerNetwork())@@ -147,9 +163,7 @@ public class EcrRegistryManager {try {CreateContainerResponse created = dockerClient.createContainerCmd(image).withName(name)- .withEnv(- "REGISTRY_STORAGE_DELETE_ENABLED=true",- "REGISTRY_HTTP_ADDR=0.0.0.0:" + CONTAINER_INTERNAL_PORT)+ .withEnv(env).withExposedPorts(exposed).withHostConfig(hostConfig).exec();diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/MskController.java b/src/main/java/io/github/hectorvent/floci/services/msk/MskController.javanew file mode 100644index 0000000000..af920df7b9--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/msk/MskController.java@@ -0,0 +1,80 @@+package io.github.hectorvent.floci.services.msk;++import io.github.hectorvent.floci.services.msk.model.MskCluster;+import jakarta.inject.Inject;+import jakarta.ws.rs.*;+import jakarta.ws.rs.core.MediaType;+import jakarta.ws.rs.core.Response;+import java.util.Map;++@Path("/")+@Produces(MediaType.APPLICATION_JSON)+@Consumes(MediaType.APPLICATION_JSON)+public class MskController {++ private final MskService mskService;++ @Inject+ public MskController(MskService mskService) {+ this.mskService = mskService;+ }++ @POST+ @Path("/v1/clusters")+ public Response createCluster(Map<String, Object> request) {+ String clusterName = (String) request.get("clusterName");+ MskCluster cluster = mskService.createCluster(clusterName);+ return Response.ok(Map.of("clusterArn", cluster.getClusterArn(), "clusterName", cluster.getClusterName(), "state", cluster.getState())).build();+ }++ @POST+ @Path("/api/v2/clusters")+ public Response createClusterV2(Map<String, Object> request) {+ // Simple mapping to V1 for now+ String clusterName = (String) request.get("clusterName");+ MskCluster cluster = mskService.createCluster(clusterName);+ return Response.ok(Map.of("clusterArn", cluster.getClusterArn(), "clusterName", cluster.getClusterName(), "state", cluster.getState())).build();+ }++ @GET+ @Path("/v1/clusters")+ public Response listClusters() {+ var clusters = mskService.listClusters();+ return Response.ok(Map.of("clusterInfoList", clusters)).build();+ }++ @GET+ @Path("/api/v2/clusters")+ public Response listClustersV2() {+ var clusters = mskService.listClusters();+ return Response.ok(Map.of("clusterInfoList", clusters)).build();+ }++ @GET+ @Path("/v1/clusters/{clusterArn}")+ public Response describeCluster(@PathParam("clusterArn") String clusterArn) {+ MskCluster cluster = mskService.describeCluster(clusterArn);+ return Response.ok(Map.of("clusterInfo", cluster)).build();+ }++ @GET+ @Path("/api/v2/clusters/{clusterArn}")+ public Response describeClusterV2(@PathParam("clusterArn") String clusterArn) {+ MskCluster cluster = mskService.describeCluster(clusterArn);+ return Response.ok(Map.of("clusterInfo", cluster)).build();+ }++ @DELETE+ @Path("/v1/clusters/{clusterArn}")+ public Response deleteCluster(@PathParam("clusterArn") String clusterArn) {+ mskService.deleteCluster(clusterArn);+ return Response.ok(Map.of("clusterArn", clusterArn, "state", "DELETING")).build();+ }++ @GET+ @Path("/v1/clusters/{clusterArn}/bootstrap-brokers")+ public Response getBootstrapBrokers(@PathParam("clusterArn") String clusterArn) {+ String bootstrapBrokers = mskService.getBootstrapBrokers(clusterArn);+ return Response.ok(Map.of("bootstrapBrokerString", bootstrapBrokers)).build();+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/MskService.java b/src/main/java/io/github/hectorvent/floci/services/msk/MskService.javanew file mode 100644index 0000000000..9538d4005b--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/msk/MskService.java@@ -0,0 +1,118 @@+package io.github.hectorvent.floci.services.msk;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.msk.model.ClusterState;+import io.github.hectorvent.floci.services.msk.model.MskCluster;+import com.fasterxml.jackson.core.type.TypeReference;+import jakarta.annotation.PostConstruct;+import jakarta.annotation.PreDestroy;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.Executors;+import java.util.concurrent.ScheduledExecutorService;+import java.util.concurrent.TimeUnit;++@ApplicationScoped+public class MskService {++ private static final Logger LOG = Logger.getLogger(MskService.class);+ private final StorageBackend<String, MskCluster> storage;+ private final EmulatorConfig config;+ private final RedpandaManager redpandaManager;+ private final ScheduledExecutorService poller = Executors.newSingleThreadScheduledExecutor();++ @Inject+ public MskService(StorageFactory storageFactory, EmulatorConfig config, RedpandaManager redpandaManager) {+ this.storage = storageFactory.create("msk", "msk-clusters.json", new TypeReference<Map<String, MskCluster>>() {});+ this.config = config;+ this.redpandaManager = redpandaManager;+ }++ @PostConstruct+ public void init() {+ startReadinessPoller();+ }++ @PreDestroy+ public void shutdown() {+ poller.shutdown();+ if (!config.services().msk().mock()) {+ for (MskCluster cluster : storage.scan(k -> true)) {+ redpandaManager.stopContainer(cluster);+ }+ }+ }++ public MskCluster createCluster(String clusterName) {+ if (storage.scan(k -> true).stream().anyMatch(c -> c.getClusterName().equals(clusterName))) {+ throw new AwsException("ConflictException", "Cluster already exists: " + clusterName, 409);+ }++ String clusterArn = String.format("arn:aws:kafka:%s:000000000000:cluster/%s/%s",+ "us-east-1", clusterName, java.util.UUID.randomUUID());++ MskCluster cluster = new MskCluster(clusterArn, clusterName);++ if (config.services().msk().mock()) {+ cluster.setState(ClusterState.ACTIVE);+ cluster.setBootstrapBrokers("localhost:9092");+ } else {+ redpandaManager.startContainer(cluster);+ }++ storage.put(clusterArn, cluster);+ return cluster;+ }++ public MskCluster describeCluster(String clusterArn) {+ return storage.get(clusterArn)+ .orElseThrow(() -> new AwsException("NotFoundException", "Cluster not found: " + clusterArn, 404));+ }++ public List<MskCluster> listClusters() {+ return storage.scan(k -> true);+ }++ public void deleteCluster(String clusterArn) {+ MskCluster cluster = storage.get(clusterArn)+ .orElseThrow(() -> new AwsException("NotFoundException", "Cluster not found: " + clusterArn, 404));++ cluster.setState(ClusterState.DELETING);+ if (!config.services().msk().mock()) {+ redpandaManager.stopContainer(cluster);+ }+ storage.delete(clusterArn);+ }++ public String getBootstrapBrokers(String clusterArn) {+ MskCluster cluster = describeCluster(clusterArn);+ return cluster.getBootstrapBrokers();+ }++ private void startReadinessPoller() {+ poller.scheduleAtFixedRate(() -> {+ try {+ for (MskCluster cluster : storage.scan(k -> true)) {+ if (cluster.getState() == ClusterState.CREATING && !config.services().msk().mock()) {+ if (redpandaManager.isReady(cluster)) {+ LOG.infov("MSK Cluster {0} is now ACTIVE", cluster.getClusterName());+ cluster.setState(ClusterState.ACTIVE);+ storage.put(cluster.getClusterArn(), cluster);+ }+ }+ }+ } catch (Exception e) {+ LOG.error("Error in MSK readiness poller", e);+ }+ }, 1, 2, TimeUnit.SECONDS);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java b/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.javanew file mode 100644index 0000000000..28ae0db0a2--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java@@ -0,0 +1,208 @@+package io.github.hectorvent.floci.services.msk;++import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.CreateContainerResponse;+import com.github.dockerjava.api.model.Bind;+import com.github.dockerjava.api.model.ExposedPort;+import com.github.dockerjava.api.model.HostConfig;+import com.github.dockerjava.api.model.Ports;+import com.github.dockerjava.api.model.Volume;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;+import io.github.hectorvent.floci.services.msk.model.MskCluster;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.io.IOException;+import java.net.HttpURLConnection;+import java.net.ServerSocket;+import java.net.URI;+import java.net.URL;+import java.nio.file.Files;+import java.nio.file.Path;+import java.time.Duration;+import java.util.List;+import java.util.Map;++@ApplicationScoped+public class RedpandaManager {++ private static final Logger LOG = Logger.getLogger(RedpandaManager.class);+ private static final int KAFKA_PORT = 9092;+ private static final int ADMIN_PORT = 9644;++ private final DockerClient dockerClient;+ private final ImageCacheService imageCacheService;+ private final ContainerDetector containerDetector;+ private final EmulatorConfig config;++ @Inject+ public RedpandaManager(DockerClient dockerClient,+ ImageCacheService imageCacheService,+ ContainerDetector containerDetector,+ EmulatorConfig config) {+ this.dockerClient = dockerClient;+ this.imageCacheService = imageCacheService;+ this.containerDetector = containerDetector;+ this.config = config;+ }++ public void startContainer(MskCluster cluster) {+ String image = config.services().msk().defaultImage();+ LOG.infov("Starting Redpanda container for MSK cluster: {0} using image {1}", cluster.getClusterName(), image);+ imageCacheService.ensureImageExists(image);++ String containerName = "floci-msk-" + cluster.getClusterName();++ // Ensure internal persistence path exists+ Path dataPath = Path.of(config.storage().persistentPath(), "msk", cluster.getClusterName());+ try {+ Files.createDirectories(dataPath);+ } catch (IOException e) {+ LOG.errorv("Failed to create MSK data directory: {0}", dataPath, e);+ }++ String hostPersistentPath = config.storage().hostPersistentPath();+ boolean isVolume = !hostPersistentPath.startsWith("/") && !hostPersistentPath.startsWith(".");++ HostConfig hostConfig = HostConfig.newHostConfig();+ List<String> cmd = new java.util.ArrayList<>(List.of("redpanda", "start", "--overprovisioned", "--smp", "1", "--memory", "512M", "--reserve-memory", "0M"));++ if (isVolume) {+ // Volume mode: mount the whole volume and point Redpanda to the subdirectory+ String internalMountPath = "/app/data";+ hostConfig.withBinds(new Bind(hostPersistentPath, new Volume(internalMountPath)));+ cmd.add("--data-dir");+ cmd.add(internalMountPath + "/msk/" + cluster.getClusterName());+ } else {+ // Directory mode: mount the specific subdirectory directly+ String hostDataPath = Path.of(hostPersistentPath, "msk", cluster.getClusterName())+ .toAbsolutePath().toString();+ hostConfig.withBinds(new Bind(hostDataPath, new Volume("/var/lib/redpanda/data")));+ }++ config.services().dockerNetwork()+ .or(() -> config.services().dockerNetwork())+ .ifPresent(network -> {+ if (!network.isBlank()) {+ hostConfig.withNetworkMode(network);+ }+ });++ Ports portBindings = new Ports();+ int hostKafkaPort = KAFKA_PORT;+ int hostAdminPort = ADMIN_PORT;++ if (!containerDetector.isRunningInContainer()) {+ hostKafkaPort = findFreePort();+ hostAdminPort = findFreePort();+ portBindings.bind(ExposedPort.tcp(KAFKA_PORT), Ports.Binding.bindPort(hostKafkaPort));+ portBindings.bind(ExposedPort.tcp(ADMIN_PORT), Ports.Binding.bindPort(hostAdminPort));+ hostConfig.withPortBindings(portBindings);+ }++ // Cleanup stale container+ try {+ dockerClient.removeContainerCmd(containerName).withForce(true).exec();+ } catch (Exception ignored) {}++ CreateContainerResponse container = dockerClient.createContainerCmd(image)+ .withName(containerName)+ .withExposedPorts(ExposedPort.tcp(KAFKA_PORT), ExposedPort.tcp(ADMIN_PORT))+ .withHostConfig(hostConfig)+ .withCmd(cmd)+ .exec();++ String containerId = container.getId();+ cluster.setContainerId(containerId);+ dockerClient.startContainerCmd(containerId).exec();++ String bootstrapHost;+ int bootstrapPort;+ String adminHost;+ int adminPort;++ if (!containerDetector.isRunningInContainer()) {+ bootstrapHost = "localhost";+ bootstrapPort = hostKafkaPort;+ adminHost = "localhost";+ adminPort = hostAdminPort;+ } else {+ var inspect = dockerClient.inspectContainerCmd(containerId).exec();+ String containerIp = inspect.getNetworkSettings().getNetworks().values().stream()+ .map(com.github.dockerjava.api.model.ContainerNetwork::getIpAddress)+ .filter(ip -> ip != null && !ip.isBlank())+ .findFirst()+ .orElse(inspect.getNetworkSettings().getIpAddress());+ bootstrapHost = containerIp;+ bootstrapPort = KAFKA_PORT;+ adminHost = containerIp;+ adminPort = ADMIN_PORT;+ }++ cluster.setBootstrapBrokers(bootstrapHost + ":" + bootstrapPort);+ LOG.infov("Redpanda container {0} started. Bootstrap: {1}", containerId, cluster.getBootstrapBrokers());++ // Wait for readiness in a separate thread or return and let the service handle state?+ // Service should handle the polling to CREATING -> ACTIVE transition.+ }++ public boolean isReady(MskCluster cluster) {+ String bootstrap = cluster.getBootstrapBrokers();+ if (bootstrap == null) return false;++ // We need the admin API to check readiness properly as per design doc (/ready endpoint)+ // For now, let's derive the admin URL.+ // In native mode it's localhost:hostAdminPort. In container mode it's containerIp:9644.++ String adminUrl;+ if (!containerDetector.isRunningInContainer()) {+ // We didn't store adminPort in cluster object, let's inspect again or store it.+ // Let's store it in cluster for simplicity during development.+ // Actually, let's just inspect the container to find the binding for 9644.+ var inspect = dockerClient.inspectContainerCmd(cluster.getContainerId()).exec();+ var bindings = inspect.getNetworkSettings().getPorts().getBindings();+ var binding = bindings.get(ExposedPort.tcp(ADMIN_PORT));+ if (binding != null && binding.length > 0) {+ adminUrl = "http://localhost:" + binding[0].getHostPortSpec() + "/ready";+ } else {+ return false;+ }+ } else {+ String containerIp = bootstrap.split(":")[0];+ adminUrl = "http://" + containerIp + ":" + ADMIN_PORT + "/ready";+ }++ try {+ HttpURLConnection conn = (HttpURLConnection) URI.create(adminUrl).toURL().openConnection();+ conn.setRequestMethod("GET");+ conn.setConnectTimeout(1000);+ conn.setReadTimeout(1000);+ return conn.getResponseCode() == 200;+ } catch (Exception e) {+ return false;+ }+ }++ public void stopContainer(MskCluster cluster) {+ if (cluster.getContainerId() == null) return;+ try {+ dockerClient.stopContainerCmd(cluster.getContainerId()).withTimeout(5).exec();+ dockerClient.removeContainerCmd(cluster.getContainerId()).withForce(true).exec();+ LOG.infov("Redpanda container {0} stopped and removed", cluster.getContainerId());+ } catch (Exception e) {+ LOG.warnv("Failed to stop Redpanda container {0}: {1}", cluster.getContainerId(), e.getMessage());+ }+ }++ private static int findFreePort() {+ try (ServerSocket s = new ServerSocket(0)) {+ s.setReuseAddress(true);+ return s.getLocalPort();+ } catch (IOException e) {+ throw new RuntimeException("Could not find a free port for Redpanda", e);+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/model/ClusterState.java b/src/main/java/io/github/hectorvent/floci/services/msk/model/ClusterState.javanew file mode 100644index 0000000000..b00db691f5--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/msk/model/ClusterState.java@@ -0,0 +1,24 @@+package io.github.hectorvent.floci.services.msk.model;++import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++@RegisterForReflection+public enum ClusterState {+ @JsonProperty("ACTIVE")+ ACTIVE,+ @JsonProperty("CREATING")+ CREATING,+ @JsonProperty("DELETING")+ DELETING,+ @JsonProperty("FAILED")+ FAILED,+ @JsonProperty("HEALING")+ HEALING,+ @JsonProperty("MAINTENANCE")+ MAINTENANCE,+ @JsonProperty("REBOOTING_BROKER")+ REBOOTING_BROKER,+ @JsonProperty("UPDATING")+ UPDATING+}diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/model/MskCluster.java b/src/main/java/io/github/hectorvent/floci/services/msk/model/MskCluster.javanew file mode 100644index 0000000000..c2ccddd2e1--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/msk/model/MskCluster.java@@ -0,0 +1,82 @@+package io.github.hectorvent.floci.services.msk.model;++import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;+import java.time.Instant;+import java.util.Map;++@RegisterForReflection+public class MskCluster {++ @JsonProperty("clusterArn")+ private String clusterArn;++ @JsonProperty("clusterName")+ private String clusterName;++ @JsonProperty("state")+ private ClusterState state;++ @JsonProperty("creationTime")+ private Instant creationTime;++ @JsonProperty("currentVersion")+ private String currentVersion;++ @JsonProperty("numberOfBrokerNodes")+ private int numberOfBrokerNodes;++ @JsonProperty("tags")+ private Map<String, String> tags;++ @JsonProperty("zookeeperConnectString")+ private String zookeeperConnectString;++ // Internal field, not directly in AWS response but needed for GetBootstrapBrokers+ private String bootstrapBrokers;++ // Docker container ID for mock=false+ private String containerId;++ public MskCluster() {}++ public MskCluster(String clusterArn, String clusterName) {+ this.clusterArn = clusterArn;+ this.clusterName = clusterName;+ this.state = ClusterState.CREATING;+ this.creationTime = Instant.now();+ this.currentVersion = "K3V6I1"; // Example version+ this.numberOfBrokerNodes = 1;+ this.zookeeperConnectString = "localhost:2181"; // Mock ZK+ }++ public String getClusterArn() { return clusterArn; }+ public void setClusterArn(String clusterArn) { this.clusterArn = clusterArn; }++ public String getClusterName() { return clusterName; }+ public void setClusterName(String clusterName) { this.clusterName = clusterName; }++ public ClusterState getState() { return state; }+ public void setState(ClusterState state) { this.state = state; }++ public Instant getCreationTime() { return creationTime; }+ public void setCreationTime(Instant creationTime) { this.creationTime = creationTime; }++ public String getCurrentVersion() { return currentVersion; }+ public void setCurrentVersion(String currentVersion) { this.currentVersion = currentVersion; }++ public int getNumberOfBrokerNodes() { return numberOfBrokerNodes; }+ public void setNumberOfBrokerNodes(int numberOfBrokerNodes) { this.numberOfBrokerNodes = numberOfBrokerNodes; }++ public Map<String, String> getTags() { return tags; }+ public void setTags(Map<String, String> tags) { this.tags = tags; }++ public String getZookeeperConnectString() { return zookeeperConnectString; }+ public void setZookeeperConnectString(String zookeeperConnectString) { this.zookeeperConnectString = zookeeperConnectString; }++ public String getBootstrapBrokers() { return bootstrapBrokers; }+ public void setBootstrapBrokers(String bootstrapBrokers) { this.bootstrapBrokers = bootstrapBrokers; }++ public String getContainerId() { return containerId; }+ public void setContainerId(String containerId) { this.containerId = containerId; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.javaindex c800ac2064..dbf5bffc2f 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java@@ -10,10 +10,12 @@ import io.github.hectorvent.floci.services.rds.model.DatabaseEngine;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.async.ResultCallback;import com.github.dockerjava.api.command.CreateContainerResponse;+import com.github.dockerjava.api.model.Bind;import com.github.dockerjava.api.model.ExposedPort;import com.github.dockerjava.api.model.Frame;import com.github.dockerjava.api.model.HostConfig;import com.github.dockerjava.api.model.Ports;+import com.github.dockerjava.api.model.Volume;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -21,6 +23,7 @@ import org.jboss.logging.Logger;import java.io.IOException;import java.net.ServerSocket;import java.nio.charset.StandardCharsets;+import java.nio.file.Path;import java.time.LocalDate;import java.time.format.DateTimeFormatter;import java.util.ArrayList;@@ -70,7 +73,37 @@ public class RdsContainerManager {int enginePort = engine.defaultPort();+ String hostPersistentPath = config.storage().hostPersistentPath();+ boolean isVolume = !hostPersistentPath.startsWith("/") && !hostPersistentPath.startsWith(".");+HostConfig hostConfig = buildHostConfig(enginePort);+ List<String> envVars = buildEnvVars(engine, masterUsername, masterPassword, dbName);++ if (isVolume) {+ String internalMountPath = "/app/data";+ hostConfig.withBinds(new Bind(hostPersistentPath, new Volume(internalMountPath)));+ String dataPath = internalMountPath + "/rds/" + instanceId;+ if (engine == DatabaseEngine.POSTGRES) {+ envVars.add("PGDATA=" + dataPath);+ } else {+ // MySQL / MariaDB typically use /var/lib/mysql+ // For volume mode, we can't easily remount a subpath to /var/lib/mysql+ // but we can try to change the engine's data dir via CMD if needed.+ // For now, let's use Bind for directory mode and warn for Volume mode if not Postgres.+ LOG.warnv("Volume-based persistence for RDS engine {0} is currently only optimized for POSTGRES. Using default non-persistent path.", engine);+ }+ } else {+ String dataPath = Path.of(config.storage().persistentPath(), "rds", instanceId).toAbsolutePath().toString();+ try {+ java.nio.file.Files.createDirectories(Path.of(dataPath));+ String hostDataPath = Path.of(hostPersistentPath, "rds", instanceId).toAbsolutePath().toString();+ String target = (engine == DatabaseEngine.POSTGRES) ? "/var/lib/postgresql/data" : "/var/lib/mysql";+ hostConfig.withBinds(new Bind(hostDataPath, new Volume(target)));+ } catch (IOException e) {+ LOG.errorv("Failed to create RDS data directory: {0}", dataPath, e);+ }+ }+String containerName = "floci-rds-" + instanceId;config.services().rds().dockerNetwork()@@ -90,7 +123,6 @@ public class RdsContainerManager {// No existing container — normal path}- List<String> envVars = buildEnvVars(engine, masterUsername, masterPassword, dbName);List<String> cmd = buildContainerCmd(engine);var createCmd = dockerClient.createContainerCmd(image)diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 39d6dbf94e..0875ed7fbe 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -27,6 +27,7 @@ floci:# Supported modes: memory, persistent, hybrid, walmode: memorypersistent-path: ./data+ host-persistent-path: ./datawal:compaction-interval-ms: 30000services:@@ -86,6 +87,10 @@ floci:iam:enabled: trueenforcement-enabled: false+ msk:+ enabled: true+ mock: false+ default-image: "redpandadata/redpanda:latest"elasticache:enabled: trueproxy-base-port: 6379diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/HealthControllerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/HealthControllerIntegrationTest.javaindex 25edb639c6..764f7341e2 100644--- a/src/test/java/io/github/hectorvent/floci/lifecycle/HealthControllerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/HealthControllerIntegrationTest.java@@ -23,6 +23,7 @@ class HealthControllerIntegrationTest {"lambda": "running","apigateway": "running","iam": "running",+ "kafka": "running","elasticache": "running","rds": "running","events": "running",diff --git a/src/test/java/io/github/hectorvent/floci/services/msk/MskServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/msk/MskServiceTest.javanew file mode 100644index 0000000000..d547c68d26--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/msk/MskServiceTest.java@@ -0,0 +1,72 @@+package io.github.hectorvent.floci.services.msk;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.msk.model.ClusterState;+import io.github.hectorvent.floci.services.msk.model.MskCluster;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.mockito.Mockito;++import java.util.List;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.Mockito.when;++class MskServiceTest {++ private MskService mskService;+ private StorageFactory storageFactory;+ private EmulatorConfig config;+ private RedpandaManager redpandaManager;++ @BeforeEach+ void setUp() {+ storageFactory = Mockito.mock(StorageFactory.class);+ when(storageFactory.create(Mockito.anyString(), Mockito.anyString(), Mockito.any()))+ .thenReturn(new InMemoryStorage<>());++ config = Mockito.mock(EmulatorConfig.class);+ var servicesConfig = Mockito.mock(EmulatorConfig.ServicesConfig.class);+ var mskConfig = Mockito.mock(EmulatorConfig.MskServiceConfig.class);++ when(config.services()).thenReturn(servicesConfig);+ when(servicesConfig.msk()).thenReturn(mskConfig);+ when(mskConfig.mock()).thenReturn(true); // Mock mode for unit tests++ redpandaManager = Mockito.mock(RedpandaManager.class);+ mskService = new MskService(storageFactory, config, redpandaManager);+ }++ @Test+ void createCluster() {+ MskCluster cluster = mskService.createCluster("test-cluster");+ assertNotNull(cluster);+ assertEquals("test-cluster", cluster.getClusterName());+ assertEquals(ClusterState.ACTIVE, cluster.getState());+ assertTrue(cluster.getClusterArn().contains("test-cluster"));+ }++ @Test+ void describeCluster() {+ MskCluster created = mskService.createCluster("test-cluster");+ MskCluster described = mskService.describeCluster(created.getClusterArn());+ assertEquals(created.getClusterArn(), described.getClusterArn());+ }++ @Test+ void listClusters() {+ mskService.createCluster("cluster-1");+ mskService.createCluster("cluster-2");+ List<MskCluster> clusters = mskService.listClusters();+ assertEquals(2, clusters.size());+ }++ @Test+ void deleteCluster() {+ MskCluster cluster = mskService.createCluster("test-cluster");+ mskService.deleteCluster(cluster.getClusterArn());+ assertTrue(mskService.listClusters().isEmpty());+ }+}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 23b36302a8..7e87caed44 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -67,6 +67,10 @@ floci:iam:enabled: trueenforcement-enabled: false+ msk:+ enabled: true+ mock: true+ default-image: "redpandadata/redpanda:latest"elasticache:enabled: trueproxy-base-port: 6379