Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(msk): advertise externally reachable broker address (#1239)

Fábio Silva61c3a07199e5
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 78d865a07a..1e799d4e46 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -599,6 +599,12 @@ public interface EmulatorConfig {
@WithDefault("redpandadata/redpanda:latest")
String defaultImage();
+
+ @WithDefault("9300")
+ int kafkaHostPortBase();
+
+ @WithDefault("9399")
+ int kafkaHostPortMax();
}
interface ElastiCacheServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java b/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java
index 4b18541d4d..74cfbe3cd8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java
@@ -10,6 +10,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.E
import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
+import io.github.hectorvent.floci.core.common.docker.PortAllocator;
import io.github.hectorvent.floci.services.msk.model.MskCluster;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
@@ -40,6 +41,7 @@ public class RedpandaManager {
private final ContainerDetector containerDetector;
private final EmulatorConfig config;
private final RegionResolver regionResolver;
+ private final PortAllocator portAllocator;
private final Map<String, Closeable> logStreams = new ConcurrentHashMap<>();
@Inject
@@ -48,13 +50,15 @@ public class RedpandaManager {
ContainerLogStreamer logStreamer,
ContainerDetector containerDetector,
EmulatorConfig config,
- RegionResolver regionResolver) {
+ RegionResolver regionResolver,
+ PortAllocator portAllocator) {
this.containerBuilder = containerBuilder;
this.lifecycleManager = lifecycleManager;
this.logStreamer = logStreamer;
this.containerDetector = containerDetector;
this.config = config;
this.regionResolver = regionResolver;
+ this.portAllocator = portAllocator;
}
public void startContainer(MskCluster cluster) {
@@ -66,10 +70,32 @@ public class RedpandaManager {
// Cleanup stale container
lifecycleManager.removeIfExists(containerName);
+ // Redpanda embeds broker addresses in Kafka protocol responses (Metadata,
+ // FindCoordinator, ...) and clients reconnect directly to whatever address
+ // it advertises. Without --advertise-kafka-addr, Redpanda advertises its own
+ // self-perceived loopback (127.0.0.1:9092), which no client outside the
+ // container can ever reach — so we must resolve and pass the externally
+ // reachable address up front. The host port has to be known before the
+ // container starts (the flag is a startup arg), so in native mode we
+ // pre-allocate it ourselves instead of letting Docker assign one dynamically.
+ String kafkaAdvertiseAddr;
+ int kafkaHostPort = 0;
+ if (!containerDetector.isRunningInContainer()) {
+ kafkaHostPort = portAllocator.allocate(
+ config.services().msk().kafkaHostPortBase(),
+ config.services().msk().kafkaHostPortMax());
+ kafkaAdvertiseAddr = "localhost:" + kafkaHostPort;
+ } else {
+ // Sibling containers on the docker network resolve the broker via its
+ // container name through Docker's embedded DNS.
+ kafkaAdvertiseAddr = containerName + ":" + KAFKA_PORT;
+ }
+
// Build command
List<String> cmd = new ArrayList<>(List.of(
"redpanda", "start", "--overprovisioned", "--smp", "1",
- "--memory", "512M", "--reserve-memory", "0M"));
+ "--memory", "512M", "--reserve-memory", "0M",
+ "--advertise-kafka-addr", kafkaAdvertiseAddr));
// Build container spec. Publish Kafka/admin ports to the host only in
// native mode; in Docker mode producers/consumers reach the broker via
@@ -80,7 +106,7 @@ public class RedpandaManager {
.withLogRotation();
if (!containerDetector.isRunningInContainer()) {
- specBuilder.withDynamicPort(KAFKA_PORT).withDynamicPort(ADMIN_PORT);
+ specBuilder.withPortBinding(KAFKA_PORT, kafkaHostPort).withDynamicPort(ADMIN_PORT);
} else {
specBuilder.withExposedPort(KAFKA_PORT).withExposedPort(ADMIN_PORT);
}
@@ -102,7 +128,15 @@ public class RedpandaManager {
ContainerSpec spec = specBuilder.build();
// Create and start container
- ContainerInfo info = lifecycleManager.createAndStart(spec);
+ ContainerInfo info;
+ try {
+ info = lifecycleManager.createAndStart(spec);
+ } catch (RuntimeException e) {
+ if (!containerDetector.isRunningInContainer()) {
+ portAllocator.release(kafkaHostPort);
+ }
+ throw e;
+ }
cluster.setContainerId(info.containerId());
// Resolve endpoints
@@ -170,6 +204,27 @@ public class RedpandaManager {
lifecycleManager.stopAndRemove(cluster.getContainerId(), logHandle);
LOG.infov("Redpanda container {0} stopped and removed", cluster.getContainerId());
+
+ releaseKafkaHostPort(cluster);
+ }
+
+ private void releaseKafkaHostPort(MskCluster cluster) {
+ if (containerDetector.isRunningInContainer()) {
+ return;
+ }
+ String bootstrap = cluster.getBootstrapBrokers();
+ if (bootstrap == null) {
+ return;
+ }
+ int separatorIndex = bootstrap.lastIndexOf(':');
+ if (separatorIndex < 0) {
+ return;
+ }
+ try {
+ portAllocator.release(Integer.parseInt(bootstrap.substring(separatorIndex + 1)));
+ } catch (NumberFormatException e) {
+ // bootstrap string not in host:port form - nothing to release
+ }
}
public void removeClusterStorage(MskCluster cluster) {
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 984b2045f9..2ae382712d 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -201,6 +201,8 @@ floci:
enabled: true
mock: false
default-image: "redpandadata/redpanda:latest"
+ kafka-host-port-base: 9300
+ kafka-host-port-max: 9399
elasticache:
enabled: true
proxy-base-port: 6379
diff --git a/src/test/java/io/github/hectorvent/floci/services/msk/RedpandaManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/msk/RedpandaManagerTest.java
index 9ca39204ff..acc2ab0cc6 100644
--- a/src/test/java/io/github/hectorvent/floci/services/msk/RedpandaManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/msk/RedpandaManagerTest.java
@@ -8,30 +8,47 @@ import com.github.dockerjava.api.model.Ports;
import com.sun.net.httpserver.HttpServer;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;
import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
+import io.github.hectorvent.floci.core.common.docker.PortAllocator;
import io.github.hectorvent.floci.services.msk.model.MskCluster;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.net.InetSocketAddress;
+import java.nio.file.Path;
import java.util.Map;
+import java.util.Optional;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class RedpandaManagerTest {
- @Mock
- private ContainerBuilder containerBuilder;
+ private static final int KAFKA_PORT = 9092;
+
@Mock
private ContainerLifecycleManager lifecycleManager;
@Mock
@@ -39,12 +56,52 @@ class RedpandaManagerTest {
@Mock
private ContainerDetector containerDetector;
@Mock
+ private RegionResolver regionResolver;
+ @Mock
+ private PortAllocator portAllocator;
+ @Mock
private EmulatorConfig config;
@Mock
- private RegionResolver regionResolver;
+ private DockerHostResolver dockerHostResolver;
+ @Mock
+ private EmbeddedDnsServer embeddedDnsServer;
+
+ @TempDir
+ Path tempDir;
+
+ RedpandaManager manager;
private HttpServer adminServer;
+ @BeforeEach
+ void setUp() {
+ EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);
+ EmulatorConfig.MskServiceConfig msk = mock(EmulatorConfig.MskServiceConfig.class);
+ EmulatorConfig.DockerConfig docker = mock(EmulatorConfig.DockerConfig.class);
+ EmulatorConfig.StorageConfig storage = mock(EmulatorConfig.StorageConfig.class);
+
+ lenient().when(config.services()).thenReturn(services);
+ lenient().when(services.msk()).thenReturn(msk);
+ lenient().when(services.dockerNetwork()).thenReturn(Optional.empty());
+ lenient().when(msk.defaultImage()).thenReturn("redpandadata/redpanda:latest");
+ lenient().when(msk.kafkaHostPortBase()).thenReturn(9300);
+ lenient().when(msk.kafkaHostPortMax()).thenReturn(9399);
+ lenient().when(config.docker()).thenReturn(docker);
+ lenient().when(docker.logMaxSize()).thenReturn("10m");
+ lenient().when(docker.logMaxFile()).thenReturn("3");
+ lenient().when(config.storage()).thenReturn(storage);
+ lenient().when(storage.hostPersistentPath()).thenReturn(tempDir.toAbsolutePath().toString());
+
+ lenient().when(embeddedDnsServer.getServerIp()).thenReturn(Optional.empty());
+
+ ContainerBuilder containerBuilder = new ContainerBuilder(config, dockerHostResolver, embeddedDnsServer);
+ manager = new RedpandaManager(containerBuilder, lifecycleManager, logStreamer, containerDetector,
+ config, regionResolver, portAllocator);
+
+ lenient().when(logStreamer.generateLogStreamName(any())).thenReturn("log-stream");
+ lenient().when(regionResolver.getDefaultRegion()).thenReturn("us-east-1");
+ }
+
@AfterEach
void tearDown() {
if (adminServer != null) {
@@ -52,8 +109,10 @@ class RedpandaManagerTest {
}
}
- private static MskCluster newCluster() {
- return new MskCluster("arn:aws:kafka:us-east-1:000000000000:cluster/test-cluster/abc", "test-cluster", "3.6.0");
+ private MskCluster newCluster() {
+ MskCluster cluster = new MskCluster("arn:aws:kafka:us-east-1:000000000000:cluster/test-cluster/abc", "test-cluster", "3.6.0");
+ cluster.setVolumeId("abc123");
+ return cluster;
}
private int startFakeAdminServer() throws Exception {
@@ -87,9 +146,6 @@ class RedpandaManagerTest {
ExposedPort.tcp(RedpandaManager.ADMIN_PORT),
new Ports.Binding[] { new Ports.Binding("0.0.0.0", String.valueOf(adminHostPort)) }));
- RedpandaManager manager = new RedpandaManager(
- containerBuilder, lifecycleManager, logStreamer, containerDetector, config, regionResolver);
-
MskCluster cluster = newCluster();
cluster.setContainerId("container-id");
cluster.setBootstrapBrokers("localhost:19092");
@@ -98,4 +154,67 @@ class RedpandaManagerTest {
"isReady() should report ready once /v1/status/ready answers 200; "
+ "if it regresses to polling /ready (which always 404s), this assertion fails");
}
+
+ @Test
+ void nativeModeAdvertisesPreAllocatedLocalhostAddress() {
+ when(containerDetector.isRunningInContainer()).thenReturn(false);
+ when(portAllocator.allocate(9300, 9399)).thenReturn(54321);
+
+ ContainerInfo info = new ContainerInfo("container-123",
+ Map.of(KAFKA_PORT, new EndpointInfo("localhost", 54321)));
+ when(lifecycleManager.createAndStart(any())).thenReturn(info);
+
+ ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
+
+ manager.startContainer(newCluster());
+
+ verify(lifecycleManager).createAndStart(specCaptor.capture());
+ ContainerSpec spec = specCaptor.getValue();
+
+ int flagIndex = spec.cmd().indexOf("--advertise-kafka-addr");
+ assertTrue(flagIndex >= 0, "cmd should contain --advertise-kafka-addr");
+ assertEquals("localhost:54321", spec.cmd().get(flagIndex + 1));
+
+ assertEquals(Integer.valueOf(54321), spec.portBindings().get(KAFKA_PORT));
+ }
+
+ @Test
+ void containerModeAdvertisesContainerNameAddress() {
+ when(containerDetector.isRunningInContainer()).thenReturn(true);
+
+ ContainerInfo info = new ContainerInfo("container-456",
+ Map.of(KAFKA_PORT, new EndpointInfo("172.18.0.5", KAFKA_PORT)));
+ when(lifecycleManager.createAndStart(any())).thenReturn(info);
+
+ ArgumentCaptor<ContainerSpec> specCaptor = ArgumentCaptor.forClass(ContainerSpec.class);
+
+ manager.startContainer(newCluster());
+
+ verify(lifecycleManager).createAndStart(specCaptor.capture());
+ ContainerSpec spec = specCaptor.getValue();
+
+ int flagIndex = spec.cmd().indexOf("--advertise-kafka-addr");
+ assertTrue(flagIndex >= 0, "cmd should contain --advertise-kafka-addr");
+ assertEquals("floci-msk-test-cluster:9092", spec.cmd().get(flagIndex + 1));
+
+ assertFalse(spec.portBindings().containsKey(KAFKA_PORT), "container mode should not publish ports to host");
+ assertTrue(spec.exposedPorts().contains(KAFKA_PORT));
+ assertTrue(spec.exposedPorts().contains(RedpandaManager.ADMIN_PORT));
+
+ verifyNoInteractions(portAllocator);
+ }
+
+ @Test
+ void stopContainerReleasesAllocatedKafkaHostPort() {
+ when(containerDetector.isRunningInContainer()).thenReturn(false);
+
+ MskCluster cluster = newCluster();
+ cluster.setContainerId("container-id");
+ cluster.setBootstrapBrokers("localhost:54321");
+
+ manager.stopContainer(cluster);
+
+ verify(lifecycleManager).stopAndRemove("container-id", null);
+ verify(portAllocator).release(54321);
+ }
}