Light, fluffy, and always free - The AWS Local Emulator alternative
fix(msk): advertise externally reachable broker address (#1239)
Fábio Silva61c3a07199e5
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 78d865a07a..1e799d4e46 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -599,6 +599,12 @@ public interface EmulatorConfig {@WithDefault("redpandadata/redpanda:latest")String defaultImage();++ @WithDefault("9300")+ int kafkaHostPortBase();++ @WithDefault("9399")+ int kafkaHostPortMax();}interface ElastiCacheServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java b/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.javaindex 4b18541d4d..74cfbe3cd8 100644--- a/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java@@ -10,6 +10,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.Eimport io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;+import io.github.hectorvent.floci.core.common.docker.PortAllocator;import io.github.hectorvent.floci.services.msk.model.MskCluster;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;@@ -40,6 +41,7 @@ public class RedpandaManager {private final ContainerDetector containerDetector;private final EmulatorConfig config;private final RegionResolver regionResolver;+ private final PortAllocator portAllocator;private final Map<String, Closeable> logStreams = new ConcurrentHashMap<>();@Inject@@ -48,13 +50,15 @@ public class RedpandaManager {ContainerLogStreamer logStreamer,ContainerDetector containerDetector,EmulatorConfig config,- RegionResolver regionResolver) {+ RegionResolver regionResolver,+ PortAllocator portAllocator) {this.containerBuilder = containerBuilder;this.lifecycleManager = lifecycleManager;this.logStreamer = logStreamer;this.containerDetector = containerDetector;this.config = config;this.regionResolver = regionResolver;+ this.portAllocator = portAllocator;}public void startContainer(MskCluster cluster) {@@ -66,10 +70,32 @@ public class RedpandaManager {// Cleanup stale containerlifecycleManager.removeIfExists(containerName);+ // Redpanda embeds broker addresses in Kafka protocol responses (Metadata,+ // FindCoordinator, ...) and clients reconnect directly to whatever address+ // it advertises. Without --advertise-kafka-addr, Redpanda advertises its own+ // self-perceived loopback (127.0.0.1:9092), which no client outside the+ // container can ever reach — so we must resolve and pass the externally+ // reachable address up front. The host port has to be known before the+ // container starts (the flag is a startup arg), so in native mode we+ // pre-allocate it ourselves instead of letting Docker assign one dynamically.+ String kafkaAdvertiseAddr;+ int kafkaHostPort = 0;+ if (!containerDetector.isRunningInContainer()) {+ kafkaHostPort = portAllocator.allocate(+ config.services().msk().kafkaHostPortBase(),+ config.services().msk().kafkaHostPortMax());+ kafkaAdvertiseAddr = "localhost:" + kafkaHostPort;+ } else {+ // Sibling containers on the docker network resolve the broker via its+ // container name through Docker's embedded DNS.+ kafkaAdvertiseAddr = containerName + ":" + KAFKA_PORT;+ }+// Build commandList<String> cmd = new ArrayList<>(List.of("redpanda", "start", "--overprovisioned", "--smp", "1",- "--memory", "512M", "--reserve-memory", "0M"));+ "--memory", "512M", "--reserve-memory", "0M",+ "--advertise-kafka-addr", kafkaAdvertiseAddr));// Build container spec. Publish Kafka/admin ports to the host only in// native mode; in Docker mode producers/consumers reach the broker via@@ -80,7 +106,7 @@ public class RedpandaManager {.withLogRotation();if (!containerDetector.isRunningInContainer()) {- specBuilder.withDynamicPort(KAFKA_PORT).withDynamicPort(ADMIN_PORT);+ specBuilder.withPortBinding(KAFKA_PORT, kafkaHostPort).withDynamicPort(ADMIN_PORT);} else {specBuilder.withExposedPort(KAFKA_PORT).withExposedPort(ADMIN_PORT);}@@ -102,7 +128,15 @@ public class RedpandaManager {ContainerSpec spec = specBuilder.build();// Create and start container- ContainerInfo info = lifecycleManager.createAndStart(spec);+ ContainerInfo info;+ try {+ info = lifecycleManager.createAndStart(spec);+ } catch (RuntimeException e) {+ if (!containerDetector.isRunningInContainer()) {+ portAllocator.release(kafkaHostPort);+ }+ throw e;+ }cluster.setContainerId(info.containerId());// Resolve endpoints@@ -170,6 +204,27 @@ public class RedpandaManager {lifecycleManager.stopAndRemove(cluster.getContainerId(), logHandle);LOG.infov("Redpanda container {0} stopped and removed", cluster.getContainerId());++ releaseKafkaHostPort(cluster);+ }++ private void releaseKafkaHostPort(MskCluster cluster) {+ if (containerDetector.isRunningInContainer()) {+ return;+ }+ String bootstrap = cluster.getBootstrapBrokers();+ if (bootstrap == null) {+ return;+ }+ int separatorIndex = bootstrap.lastIndexOf(':');+ if (separatorIndex < 0) {+ return;+ }+ try {+ portAllocator.release(Integer.parseInt(bootstrap.substring(separatorIndex + 1)));+ } catch (NumberFormatException e) {+ // bootstrap string not in host:port form - nothing to release+ }}public void removeClusterStorage(MskCluster cluster) {diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 984b2045f9..2ae382712d 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -201,6 +201,8 @@ floci:enabled: truemock: falsedefault-image: "redpandadata/redpanda:latest"+ kafka-host-port-base: 9300+ kafka-host-port-max: 9399elasticache:enabled: trueproxy-base-port: 6379diff --git a/src/test/java/io/github/hectorvent/floci/services/msk/RedpandaManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/msk/RedpandaManagerTest.javaindex 9ca39204ff..acc2ab0cc6 100644--- a/src/test/java/io/github/hectorvent/floci/services/msk/RedpandaManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/msk/RedpandaManagerTest.java@@ -8,30 +8,47 @@ import com.github.dockerjava.api.model.Ports;import com.sun.net.httpserver.HttpServer;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerDetector;import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;+import io.github.hectorvent.floci.core.common.docker.PortAllocator;import io.github.hectorvent.floci.services.msk.model.MskCluster;import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.extension.ExtendWith;+import org.junit.jupiter.api.io.TempDir;+import org.mockito.ArgumentCaptor;import org.mockito.Mock;import org.mockito.junit.jupiter.MockitoExtension;import java.net.InetSocketAddress;+import java.nio.file.Path;import java.util.Map;+import java.util.Optional;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.lenient;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.verifyNoInteractions;import static org.mockito.Mockito.when;@ExtendWith(MockitoExtension.class)class RedpandaManagerTest {- @Mock- private ContainerBuilder containerBuilder;+ private static final int KAFKA_PORT = 9092;+@Mockprivate ContainerLifecycleManager lifecycleManager;@Mock@@ -39,12 +56,52 @@ class RedpandaManagerTest {@Mockprivate ContainerDetector containerDetector;@Mock+ private RegionResolver regionResolver;+ @Mock+ private PortAllocator portAllocator;+ @Mockprivate EmulatorConfig config;@Mock- private RegionResolver regionResolver;+ private DockerHostResolver dockerHostResolver;+ @Mock+ private EmbeddedDnsServer embeddedDnsServer;++ @TempDir+ Path tempDir;++ RedpandaManager manager;private HttpServer adminServer;+ @BeforeEach+ void setUp() {+ EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);+ EmulatorConfig.MskServiceConfig msk = mock(EmulatorConfig.MskServiceConfig.class);+ EmulatorConfig.DockerConfig docker = mock(EmulatorConfig.DockerConfig.class);+ EmulatorConfig.StorageConfig storage = mock(EmulatorConfig.StorageConfig.class);++ lenient().when(config.services()).thenReturn(services);+ lenient().when(services.msk()).thenReturn(msk);+ lenient().when(services.dockerNetwork()).thenReturn(Optional.empty());+ lenient().when(msk.defaultImage()).thenReturn("redpandadata/redpanda:latest");+ lenient().when(msk.kafkaHostPortBase()).thenReturn(9300);+ lenient().when(msk.kafkaHostPortMax()).thenReturn(9399);+ lenient().when(config.docker()).thenReturn(docker);+ lenient().when(docker.logMaxSize()).thenReturn("10m");+ lenient().when(docker.logMaxFile()).thenReturn("3");+ lenient().when(config.storage()).thenReturn(storage);+ lenient().when(storage.hostPersistentPath()).thenReturn(tempDir.toAbsolutePath().toString());++ lenient().when(embeddedDnsServer.getServerIp()).thenReturn(Optional.empty());++ ContainerBuilder containerBuilder = new ContainerBuilder(config, dockerHostResolver, embeddedDnsServer);+ manager = new RedpandaManager(containerBuilder, lifecycleManager, logStreamer, containerDetector,+ config, regionResolver, portAllocator);++ lenient().when(logStreamer.generateLogStreamName(any())).thenReturn("log-stream");+ lenient().when(regionResolver.getDefaultRegion()).thenReturn("us-east-1");+ }+@AfterEachvoid tearDown() {if (adminServer != null) {@@ -52,8 +109,10 @@ class RedpandaManagerTest {}}- private static MskCluster newCluster() {- return new MskCluster("arn:aws:kafka:us-east-1:000000000000:cluster/test-cluster/abc", "test-cluster", "3.6.0");+ private MskCluster newCluster() {+ MskCluster cluster = new MskCluster("arn:aws:kafka:us-east-1:000000000000:cluster/test-cluster/abc", "test-cluster", "3.6.0");+ cluster.setVolumeId("abc123");+ return cluster;}private int startFakeAdminServer() throws Exception {@@ -87,9 +146,6 @@ class RedpandaManagerTest {ExposedPort.tcp(RedpandaManager.ADMIN_PORT),new Ports.Binding[] { new Ports.Binding("0.0.0.0", String.valueOf(adminHostPort)) }));- RedpandaManager manager = new RedpandaManager(- containerBuilder, lifecycleManager, logStreamer, containerDetector, config, regionResolver);-MskCluster cluster = newCluster();cluster.setContainerId("container-id");cluster.setBootstrapBrokers("localhost:19092");@@ -98,4 +154,67 @@ class RedpandaManagerTest {"isReady() should report ready once /v1/status/ready answers 200; "+ "if it regresses to polling /ready (which always 404s), this assertion fails");}++ @Test+ void nativeModeAdvertisesPreAllocatedLocalhostAddress() {+ when(containerDetector.isRunningInContainer()).thenReturn(false);+ when(portAllocator.allocate(9300, 9399)).thenReturn(54321);++ ContainerInfo info = new ContainerInfo("container-123",+ Map.of(KAFKA_PORT, new EndpointInfo("localhost", 54321)));+ when(lifecycleManager.createAndStart(any())).thenReturn(info);++ ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);++ manager.startContainer(newCluster());++ verify(lifecycleManager).createAndStart(specCaptor.capture());+ ContainerSpec spec = specCaptor.getValue();++ int flagIndex = spec.cmd().indexOf("--advertise-kafka-addr");+ assertTrue(flagIndex >= 0, "cmd should contain --advertise-kafka-addr");+ assertEquals("localhost:54321", spec.cmd().get(flagIndex + 1));++ assertEquals(Integer.valueOf(54321), spec.portBindings().get(KAFKA_PORT));+ }++ @Test+ void containerModeAdvertisesContainerNameAddress() {+ when(containerDetector.isRunningInContainer()).thenReturn(true);++ ContainerInfo info = new ContainerInfo("container-456",+ Map.of(KAFKA_PORT, new EndpointInfo("172.18.0.5", KAFKA_PORT)));+ when(lifecycleManager.createAndStart(any())).thenReturn(info);++ ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);++ manager.startContainer(newCluster());++ verify(lifecycleManager).createAndStart(specCaptor.capture());+ ContainerSpec spec = specCaptor.getValue();++ int flagIndex = spec.cmd().indexOf("--advertise-kafka-addr");+ assertTrue(flagIndex >= 0, "cmd should contain --advertise-kafka-addr");+ assertEquals("floci-msk-test-cluster:9092", spec.cmd().get(flagIndex + 1));++ assertFalse(spec.portBindings().containsKey(KAFKA_PORT), "container mode should not publish ports to host");+ assertTrue(spec.exposedPorts().contains(KAFKA_PORT));+ assertTrue(spec.exposedPorts().contains(RedpandaManager.ADMIN_PORT));++ verifyNoInteractions(portAllocator);+ }++ @Test+ void stopContainerReleasesAllocatedKafkaHostPort() {+ when(containerDetector.isRunningInContainer()).thenReturn(false);++ MskCluster cluster = newCluster();+ cluster.setContainerId("container-id");+ cluster.setBootstrapBrokers("localhost:54321");++ manager.stopContainer(cluster);++ verify(lifecycleManager).stopAndRemove("container-id", null);+ verify(portAllocator).release(54321);+ }}