Light, fluffy, and always free - The AWS Local Emulator alternative
fix(elbv2): do not let a listener bind a port Floci reserves for itself (#3034)
Jerald Lindbergbf517ac95245
diff --git a/src/main/java/io/github/hectorvent/floci/config/TlsProxyServer.java b/src/main/java/io/github/hectorvent/floci/config/TlsProxyServer.javaindex 84fade489f..524bb5ca3d 100644--- a/src/main/java/io/github/hectorvent/floci/config/TlsProxyServer.java+++ b/src/main/java/io/github/hectorvent/floci/config/TlsProxyServer.java@@ -16,6 +16,7 @@ import java.util.ArrayList;import java.util.LinkedHashSet;import java.util.List;import java.util.Set;+import java.util.concurrent.ConcurrentHashMap;/*** A TCP proxy server that enables HTTP and HTTPS on the same port (LocalStack parity).@@ -58,6 +59,25 @@ public class TlsProxyServer {private final int httpBackendPort;private final int httpsBackendPort;private final List<NetServer> proxyServers = new ArrayList<>();+ /**+ * Ports whose bind definitively failed. Absent means bound, or not resolved yet: callers+ * default to treating the port as the proxy's until the outcome is known, which is what stops+ * an ELBv2 listener racing the proxy for it during start-up.+ */+ // Package-private for TlsProxyServerReservedPortTest, which needs to stage a failed bind.+ final Set<Integer> failedPorts = ConcurrentHashMap.newKeySet();+ /**+ * Notified with a port the proxy has just given up on. A listener that yielded while the bind+ * was still unresolved has no other way to learn the port came free, and would otherwise stay+ * registered with nothing serving it.+ */+ private final List<java.util.function.IntConsumer> portReleasedHandlers = new java.util.concurrent.CopyOnWriteArrayList<>();++ /** Registers a callback invoked when a bind fails and the port stops being reserved. */+ public void onPortReleased(java.util.function.IntConsumer handler) {+ portReleasedHandlers.add(handler);+ failedPorts.forEach(handler::accept);+ }private NetClient client;@Inject@@ -91,12 +111,15 @@ public class TlsProxyServer {proxyServers.add(server);server.listen().onComplete(ar -> {if (ar.succeeded()) {+ failedPorts.remove(port);LOG.infov("TLS proxy: listening on port {0} (HTTP→{1}, HTTPS→{2})",String.valueOf(port), String.valueOf(httpBackendPort), String.valueOf(httpsBackendPort));} else if (port == config.port()) {LOG.errorv("TLS proxy: failed to start on public port {0}: {1}",String.valueOf(port), ar.cause().getMessage());} else {+ failedPorts.add(port);+ portReleasedHandlers.forEach(h -> h.accept(port));// The extra AWS-HTTPS port (443 by default) is privileged; binding it fails in// unprivileged environments (e.g. CI/test). Non-fatal — HTTPS on that port is// simply unavailable. Set floci.tls.aws-https-port=0 to skip the attempt.@@ -108,6 +131,18 @@ public class TlsProxyServer {}}+ /**+ * Whether the proxy holds, or is still expected to hold, {@code port}.+ *+ * <p>False once a bind has definitively failed. That is not fatal for the AWS-HTTPS port: it is+ * privileged, so binding it fails in unprivileged environments and HTTPS there is simply+ * unavailable. Anything that yields ports to the proxy has to consult this rather than+ * configuration alone, or a port the proxy never acquired ends up served by nobody.+ */+ public boolean reservesPort(int port) {+ return config.tls().enabled() && listenPorts().contains(port) && !failedPorts.contains(port);+ }+/*** The set of ports the proxy listens on: always the public Floci {@link EmulatorConfig#port()},* plus {@code floci.tls.aws-https-port} (443 by default) so AWS-style HTTPS callbacks reachdiff --git a/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2DataPlane.java b/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2DataPlane.javaindex c79f6de435..4d8d865252 100644--- a/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2DataPlane.java+++ b/src/main/java/io/github/hectorvent/floci/services/elbv2/ElbV2DataPlane.java@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.elbv2;import com.fasterxml.jackson.core.type.TypeReference;import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.config.TlsProxyServer;import io.github.hectorvent.floci.services.ec2.Ec2Service;import io.github.hectorvent.floci.services.elbv2.model.Action;import io.github.hectorvent.floci.services.elbv2.model.Listener;@@ -61,6 +62,9 @@ public class ElbV2DataPlane {@InjectEmulatorConfig config;+ @Inject+ TlsProxyServer tlsProxyServer;+@InjectLambdaService lambdaService;@@ -71,6 +75,8 @@ public class ElbV2DataPlane {ObjectMapper objectMapper;private final Map<Integer, HttpServer> servers = new ConcurrentHashMap<>();+ private final java.util.concurrent.atomic.AtomicBoolean releaseHandlerRegistered =+ new java.util.concurrent.atomic.AtomicBoolean();private final Map<Integer, Map<String, String>> listenersByHostAndPort = new ConcurrentHashMap<>();private final Map<String, ListenerBinding> listenerBindings = new ConcurrentHashMap<>();private final Map<String, AtomicReference<List<CompiledRule>>> ruleChains = new ConcurrentHashMap<>();@@ -114,6 +120,7 @@ public class ElbV2DataPlane {listenerBindings.put(listenerArn, binding);listenersByHostAndPort.computeIfAbsent(binding.port(), ignored -> new ConcurrentHashMap<>()).put(binding.host(), listenerArn);+ ensureReleaseHandlerRegistered();servers.computeIfAbsent(binding.port(), this::startPortServer);}@@ -129,6 +136,7 @@ public class ElbV2DataPlane {listenerRegions.put(listenerArn, region);listenersByHostAndPort.computeIfAbsent(newBinding.port(), ignored -> new ConcurrentHashMap<>()).put(newBinding.host(), listenerArn);+ ensureReleaseHandlerRegistered();servers.computeIfAbsent(newBinding.port(), this::startPortServer);return;}@@ -163,7 +171,71 @@ public class ElbV2DataPlane {}}+ /**+ * Ports Floci reserves for itself, which a load balancer listener must not take.+ *+ * <p>The emulator's own port is always reserved. With TLS enabled the TLS proxy also binds+ * {@code floci.tls.aws-https-port} (443 by default), because CDK's custom-resource+ * {@code cfn-response} callback hardcodes {@code https://} and ignores the port in the+ * ResponseURL, so it always lands on 443.+ *+ * <p>Without this the two raced: whichever started first won. A stack restored from+ * persistence could hand 443 to an ALB listener, which then answered CDK's callback in plain+ * HTTP and hung every {@code Custom::} resource - and the reverse on a fresh start. Yielding+ * here makes the outcome deterministic and keeps the emulator's own control path working. To+ * give 443 to load balancers instead, set {@code floci.tls.aws-https-port=0} or disable TLS.+ */+ // Package-private for ElbV2ReservedPortTest.+ boolean isReservedByFloci(int port) {+ if (port == config.port()) {+ return true;+ }+ // Ask the proxy whether it actually holds the port rather than inferring it from config.+ // Binding the AWS-HTTPS port is privileged and its failure is non-fatal, so a port the+ // proxy never acquired must stay available to listeners instead of going unserved.+ return tlsProxyServer.reservesPort(port);+ }++ /**+ * Binds any registered listener whose port the proxy has just released. Yielding a port whose+ * bind had not resolved yet is the safe default, but if that bind then fails nothing else+ * would ever retry the listener, leaving it registered with no data plane and the port free.+ */+ private void bindListenersWaitingOn(int port) {+ if (listenersByHostAndPort.containsKey(port)) {+ servers.computeIfAbsent(port, this::startPortServer);+ }+ }++ /**+ * Subscribes to the proxy's released ports, once, on the first listener to need it: the data+ * plane is only live once a listener exists, and this keeps the proxy out of the mock-mode+ * path entirely.+ *+ * <p>Placement is load-bearing on both sides. It runs after the listener is in+ * {@code listenersByHostAndPort}, so the replay of ports already given up finds it, and before+ * {@code servers.computeIfAbsent}, because {@code onPortReleased} replays synchronously: doing+ * this from inside the mapping function re-entered {@code computeIfAbsent} for the very key it+ * was still computing, which a {@link ConcurrentHashMap} rejects outright.+ */+ private void ensureReleaseHandlerRegistered() {+ if (releaseHandlerRegistered.compareAndSet(false, true)) {+ tlsProxyServer.onPortReleased(this::bindListenersWaitingOn);+ }+ }+private HttpServer startPortServer(int port) {+ if (isReservedByFloci(port)) {+ // Returning null leaves `servers` untouched: computeIfAbsent skips a null mapping.+ // The emulator's own port cannot be handed over, so the two cases need different+ // advice: freeing floci.tls.aws-https-port is only meaningful for the TLS proxy.+ String remedy = port == config.port()+ ? "This is the emulator's own port, so it cannot be freed; give the listener a different port."+ : "Set floci.tls.aws-https-port=0 (or disable TLS) to free it.";+ LOG.warnv("ELBv2 listener port {0} is reserved by Floci itself; the listener is "+ + "registered but serves no traffic. {1}", String.valueOf(port), remedy);+ return null;+ }HttpServer server = vertx.createHttpServer(new HttpServerOptions().setHost("0.0.0.0").setPort(port));diff --git a/src/test/java/io/github/hectorvent/floci/config/TlsProxyServerReservedPortTest.java b/src/test/java/io/github/hectorvent/floci/config/TlsProxyServerReservedPortTest.javanew file mode 100644index 0000000000..a536f1ae53--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/config/TlsProxyServerReservedPortTest.java@@ -0,0 +1,122 @@+package io.github.hectorvent.floci.config;++import io.vertx.core.Future;+import io.vertx.core.Promise;+import io.vertx.core.Vertx;+import io.vertx.core.net.NetClient;+import io.vertx.core.net.NetServer;+import java.util.ArrayList;+import java.util.List;+import org.junit.jupiter.api.Test;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++/**+ * The proxy only reserves a port it holds or still expects to hold. Binding the AWS-HTTPS port is+ * privileged and its failure is non-fatal, so a port the proxy never acquired has to stay+ * available: anything that yields to the proxy on configuration alone would leave it unserved.+ */+class TlsProxyServerReservedPortTest {++ private static TlsProxyServer proxyWith(int flociPort, boolean tlsEnabled, int awsHttpsPort) {+ EmulatorConfig.TlsConfig tls = mock(EmulatorConfig.TlsConfig.class);+ when(tls.enabled()).thenReturn(tlsEnabled);+ when(tls.awsHttpsPort()).thenReturn(awsHttpsPort);++ EmulatorConfig config = mock(EmulatorConfig.class);+ when(config.port()).thenReturn(flociPort);+ when(config.tls()).thenReturn(tls);++ // The listen() future is left uncompleted on purpose: that is the "bind outcome not yet+ // known" state, which must still reserve the port. Tests that need a resolved outcome+ // stage it directly on failedPorts.+ NetServer server = mock(NetServer.class);+ when(server.connectHandler(any())).thenReturn(server);+ when(server.listen()).thenReturn(Promise.<NetServer>promise().future());++ Vertx vertx = mock(Vertx.class);+ when(vertx.createNetClient()).thenReturn(mock(NetClient.class));+ when(vertx.createNetServer(any())).thenReturn(server);++ return new TlsProxyServer(vertx, config, 4510, 4511);+ }++ /**+ * Variant that resolves listen() to {@code outcome} and lets the caller register a handler+ * before construction, since the bind callback fires during the constructor.+ */+ private static TlsProxyServer proxyWith(int flociPort, boolean tlsEnabled, int awsHttpsPort,+ Future<NetServer> outcome,+ java.util.function.Consumer<TlsProxyServer> beforeStart) {+ EmulatorConfig.TlsConfig tls = mock(EmulatorConfig.TlsConfig.class);+ when(tls.enabled()).thenReturn(tlsEnabled);+ when(tls.awsHttpsPort()).thenReturn(awsHttpsPort);+ EmulatorConfig config = mock(EmulatorConfig.class);+ when(config.port()).thenReturn(flociPort);+ when(config.tls()).thenReturn(tls);++ NetServer server = mock(NetServer.class);+ when(server.connectHandler(any())).thenReturn(server);+ when(server.listen()).thenReturn(outcome);++ Vertx vertx = mock(Vertx.class);+ when(vertx.createNetClient()).thenReturn(mock(NetClient.class));+ when(vertx.createNetServer(any())).thenReturn(server);++ TlsProxyServer proxy = new TlsProxyServer(vertx, config, 4510, 4511);+ beforeStart.accept(proxy);+ return proxy;+ }++ @Test+ void reservesTheAwsHttpsPortBeforeTheBindOutcomeIsKnown() {+ TlsProxyServer proxy = proxyWith(4566, true, 443);+ assertTrue(proxy.reservesPort(443),+ "an unresolved bind must still hold the port, or a listener races the proxy for it");+ }++ @Test+ void releasesTheAwsHttpsPortOnceItsBindHasFailed() {+ TlsProxyServer proxy = proxyWith(4566, true, 443);+ proxy.failedPorts.add(443);+ assertFalse(proxy.reservesPort(443),+ "a port the proxy failed to bind must stay available to load balancer listeners");+ }++ @Test+ void reservesNothingWhenTlsIsDisabled() {+ TlsProxyServer proxy = proxyWith(4566, false, 443);+ assertFalse(proxy.reservesPort(443));+ }++ @Test+ void doesNotReserveTheAwsHttpsPortWhenItIsDisabled() {+ TlsProxyServer proxy = proxyWith(4566, true, 0);+ assertFalse(proxy.reservesPort(443));+ }++ @Test+ void notifiesHandlersWhenABindFails() {+ List<Integer> released = new ArrayList<>();+ // listen() resolving to a failure is what happens when the privileged port is refused.+ TlsProxyServer proxy = proxyWith(4566, true, 443,+ Future.failedFuture(new RuntimeException("permission denied")),+ p -> p.onPortReleased(released::add));++ assertTrue(released.contains(443),+ "a listener that yielded 443 has no other way to learn the bind failed");+ assertFalse(proxy.reservesPort(443));+ }++ @Test+ void doesNotReserveAnUnrelatedPort() {+ TlsProxyServer proxy = proxyWith(4566, true, 443);+ assertFalse(proxy.reservesPort(8080));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ReservedPortTest.java b/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ReservedPortTest.javanew file mode 100644index 0000000000..3ccf0f3f37--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/elbv2/ElbV2ReservedPortTest.java@@ -0,0 +1,138 @@+package io.github.hectorvent.floci.services.elbv2;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.config.TlsProxyServer;+import io.github.hectorvent.floci.services.elbv2.model.Listener;+import io.vertx.core.Promise;+import io.vertx.core.Vertx;+import io.vertx.core.http.HttpServer;+import io.vertx.core.http.HttpServerOptions;+import org.junit.jupiter.api.Test;++import java.util.List;+import java.util.function.IntConsumer;++import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyInt;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++/**+ * A load balancer listener must not bind a port Floci needs for itself, or the two race and+ * whichever starts first wins - see {@link ElbV2DataPlane#isReservedByFloci(int)}.+ */+class ElbV2ReservedPortTest {++ private static ElbV2DataPlane dataPlaneWith(int flociPort, boolean tlsEnabled, int awsHttpsPort) {+ EmulatorConfig.TlsConfig tls = mock(EmulatorConfig.TlsConfig.class);+ when(tls.enabled()).thenReturn(tlsEnabled);+ when(tls.awsHttpsPort()).thenReturn(awsHttpsPort);++ EmulatorConfig config = mock(EmulatorConfig.class);+ when(config.port()).thenReturn(flociPort);+ when(config.tls()).thenReturn(tls);++ // Mirrors TlsProxyServer.reservesPort for the configured case; the bind-outcome half of+ // that contract is covered by TlsProxyServerReservedPortTest.+ TlsProxyServer proxy = mock(TlsProxyServer.class);+ when(proxy.reservesPort(anyInt())).thenAnswer(inv ->+ tlsEnabled && awsHttpsPort > 0 && (int) inv.getArgument(0) == awsHttpsPort);++ ElbV2DataPlane dataPlane = new ElbV2DataPlane();+ dataPlane.config = config;+ dataPlane.tlsProxyServer = proxy;+ return dataPlane;+ }++ @Test+ void reservesTheEmulatorsOwnPortEvenWithTlsOff() {+ ElbV2DataPlane dataPlane = dataPlaneWith(4566, false, 443);+ assertTrue(dataPlane.isReservedByFloci(4566));+ }++ @Test+ void reservesTheAwsHttpsPortWhenTlsIsOn() {+ ElbV2DataPlane dataPlane = dataPlaneWith(4566, true, 443);+ assertTrue(dataPlane.isReservedByFloci(443),+ "with TLS on, 443 carries CDK's cfn-response callback");+ }++ @Test+ void leaves443ToLoadBalancersWhenTlsIsOff() {+ ElbV2DataPlane dataPlane = dataPlaneWith(4566, false, 443);+ assertFalse(dataPlane.isReservedByFloci(443));+ }++ @Test+ void leaves443ToLoadBalancersWhenTheAwsHttpsPortIsDisabled() {+ ElbV2DataPlane dataPlane = dataPlaneWith(4566, true, 0);+ assertFalse(dataPlane.isReservedByFloci(443),+ "aws-https-port=0 is the documented escape hatch");+ }++ /**+ * The proxy gave up the port before any listener existed, so the data plane subscribes and is+ * replayed that port in the same breath. The replay rebinds listeners waiting on the port,+ * which reaches back into the same server map: doing the subscribing from inside the map's+ * own mapping function re-entered computeIfAbsent for the key it was still computing, and+ * ConcurrentHashMap answers that with IllegalStateException: Recursive update.+ */+ @Test+ void aListenerStartsOnAPortTheProxyGaveUpBeforeItSubscribed() {+ EmulatorConfig.TlsConfig tls = mock(EmulatorConfig.TlsConfig.class);+ when(tls.enabled()).thenReturn(true);+ when(tls.awsHttpsPort()).thenReturn(443);+ EmulatorConfig.ElbV2ServiceConfig elbv2 = mock(EmulatorConfig.ElbV2ServiceConfig.class);+ when(elbv2.mock()).thenReturn(false);+ EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);+ when(services.elbv2()).thenReturn(elbv2);+ EmulatorConfig config = mock(EmulatorConfig.class);+ when(config.port()).thenReturn(4566);+ when(config.tls()).thenReturn(tls);+ when(config.services()).thenReturn(services);++ // 443 is already in failedPorts, so subscribing replays it synchronously, exactly as+ // TlsProxyServer.onPortReleased does.+ TlsProxyServer proxy = mock(TlsProxyServer.class);+ when(proxy.reservesPort(anyInt())).thenReturn(false);+ doAnswer(inv -> {+ inv.getArgument(0, IntConsumer.class).accept(443);+ return null;+ }).when(proxy).onPortReleased(any());++ HttpServer server = mock(HttpServer.class);+ when(server.requestHandler(any())).thenReturn(server);+ // Left uncompleted: the listen outcome is not what this test is about.+ when(server.listen()).thenReturn(Promise.<HttpServer>promise().future());+ Vertx vertx = mock(Vertx.class);+ when(vertx.createHttpServer(any(HttpServerOptions.class))).thenReturn(server);++ ElbV2DataPlane dataPlane = new ElbV2DataPlane();+ dataPlane.config = config;+ dataPlane.tlsProxyServer = proxy;+ dataPlane.vertx = vertx;+ dataPlane.elbV2Service = mock(ElbV2Service.class);++ Listener listener = new Listener();+ listener.setListenerArn("arn:aws:elasticloadbalancing:us-west-2:000000000000:listener/app/lb/1/2");+ listener.setLoadBalancerArn("arn:aws:elasticloadbalancing:us-west-2:000000000000:loadbalancer/app/lb/1");+ listener.setPort(443);++ assertDoesNotThrow(() -> dataPlane.startListener(listener, "us-west-2", List.of()),+ "the released-port replay re-entered the server map for the key it was computing");+ verify(vertx, times(1)).createHttpServer(any(HttpServerOptions.class));+ }++ @Test+ void leavesOrdinaryListenerPortsAlone() {+ ElbV2DataPlane dataPlane = dataPlaneWith(4566, true, 443);+ assertFalse(dataPlane.isReservedByFloci(80));+ assertFalse(dataPlane.isReservedByFloci(8080));+ }+}