Light, fluffy, and always free - The AWS Local Emulator alternative
feat(iot): MQTT over TLS listener on 8883 (#3142)
Mario Toffiaee2f55b78422
diff --git a/docs/configuration/tls.md b/docs/configuration/tls.mdindex 970b9b0efa..7786beb727 100644--- a/docs/configuration/tls.md+++ b/docs/configuration/tls.md@@ -123,6 +123,10 @@ wss://localhost:4566/ws/{apiId}/{stage}No additional configuration is needed: Vert.x handles TLS at the transport layer transparently.+## MQTT over TLS (8883)++When TLS is enabled, the IoT MQTT broker also listens on port 8883 with the same certificate as the HTTPS endpoint, hostnames learned at runtime included. Devices connect with `ssl://` trusting `ca.pem`; see [IoT Core](../services/iot.md#mqtt-over-tls).+## SDK Configuration ExamplesWith `AWS_CA_BUNDLE` set as in Quick Start, no SDK needs code changes. The examples below trust the CA in code instead, for processes whose environment you do not control.diff --git a/docs/services/iot.md b/docs/services/iot.mdindex fce6772ba8..2ed9b5088f 100644--- a/docs/services/iot.md+++ b/docs/services/iot.md@@ -79,11 +79,23 @@ Broker scope:- Target real AWS IoT/device SDK style MQTT clients, not only handcrafted packet tests.- Support MQTT v3 and MQTT 5 CONNECT handling used by local compatibility tests.- Support QoS 0 and QoS 1 publish/subscribe behavior for the local AWS IoT slice.-- Keep MQTT plaintext-only for this phase; TLS and mTLS are out of scope.+- Serve MQTT over TLS on 8883 next to plaintext 1883 when TLS is enabled; verifying device certificates (mutual TLS) is follow-up scope.- Keep MQTT authorization permissive for now, but leave room for a later pluggable IoT certificate and policy authorizer.- Keep MQTT broker logging minimal.- Validate the relevant IoT compatibility tests against the native binary before considering the phase complete.+### MQTT over TLS++With `FLOCI_TLS_ENABLED=true` the broker also listens on `FLOCI_SERVICES_IOT_MQTT_TLS_PORT` (default `8883`, the port AWS IoT uses for X.509 device connections; `0` disables it). It presents the same certificate as the HTTPS endpoint, issued by the Floci CA, over TLS 1.2 or 1.3. A device connects with `ssl://localhost:8883` trusting `GET /_floci/ca.pem`, as it would trust Amazon Root CA 1 against AWS IoT. A custom domain added to the server certificate at runtime (see [TLS](../configuration/tls.md#custom-domains-learned-at-runtime)) is served on 8883 from the next connection on, without a restart.++The listener asks for a client certificate and accepts the connection whether or not one is presented and whoever signed it: 8883 is as permissive as 1883 until certificate verification lands. Sessions, subscriptions and reserved topics are shared with the plaintext listener, so a client id connecting on one port replaces its session on the other, as on AWS. Both listeners start together: with the first IoT API call, or at boot with `FLOCI_SERVICES_IOT_MQTT_AUTO_START=true`.++```bash+docker run -e FLOCI_TLS_ENABLED=true -e FLOCI_SERVICES_IOT_MQTT_AUTO_START=true -p 4566:4566 -p 8883:8883 floci/floci:latest+curl http://localhost:4566/_floci/ca.pem -o ca.pem+mosquitto_sub -h localhost -p 8883 --cafile ca.pem -t 'devices/#'+```+## Reserved TopicsAWS IoT reserved topics such as `$aws/things/{thingName}/shadow/update` are service control topics, not ordinary application topics. Floci should handle these publishes by invoking IoT shadow behavior and then publishing the AWS-compatible response topics through the broker.@@ -106,7 +118,7 @@ Implementation notes:Current accepted limitation:-- Certificate and policy authorization are not enforced at the broker layer yet.+- Certificate and policy authorization are not enforced at the broker layer yet, on either port.- Persistent offline sessions are not modeled yet.- QoS 2 and advanced MQTT 5 property semantics remain follow-up scope.diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 0c1515f390..57155332a3 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -793,6 +793,13 @@ public interface EmulatorConfig {@WithDefault("1883")int port();++ /**+ * MQTT over TLS listener, the port AWS IoT serves for X.509 device connections. Opened only+ * while {@code floci.tls.enabled} is true; {@code 0} disables it. Env: FLOCI_SERVICES_IOT_MQTT_TLS_PORT+ */+ @WithDefault("8883")+ int tlsPort();}interface IotDataServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.javaindex c5aab03718..3ee4b8bde4 100644--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java@@ -5,9 +5,16 @@ import io.github.hectorvent.floci.services.iot.model.IotRetainedMessage;import io.netty.handler.codec.mqtt.MqttQoS;import io.quarkus.runtime.ShutdownEvent;import io.quarkus.runtime.StartupEvent;+import io.quarkus.tls.CertificateUpdatedEvent;+import io.quarkus.tls.TlsConfiguration;+import io.quarkus.tls.TlsConfigurationRegistry;+import io.quarkus.tls.runtime.config.TlsConfig;import io.vertx.core.Vertx;import io.vertx.core.buffer.Buffer;+import io.vertx.core.http.ClientAuth;+import io.vertx.core.net.KeyCertOptions;import io.vertx.core.net.SocketAddress;+import io.vertx.core.net.TrustOptions;import io.vertx.mqtt.MqttEndpoint;import io.vertx.mqtt.MqttServer;import io.vertx.mqtt.MqttServerOptions;@@ -20,6 +27,11 @@ import jakarta.enterprise.inject.Instance;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import javax.net.ssl.SSLEngine;+import javax.net.ssl.X509ExtendedTrustManager;+import java.net.Socket;+import java.security.cert.CertificateException;+import java.security.cert.X509Certificate;import java.util.ArrayList;import java.util.Base64;import java.util.HashSet;@@ -34,18 +46,63 @@ public class IotMqttBrokerService {private static final Logger LOG = Logger.getLogger(IotMqttBrokerService.class);+ /**+ * The TLS listener asks for a client certificate and accepts whichever one is presented: AWS+ * IoT trusts a device certificate because it is registered, not because of its issuer, and+ * that lookup belongs to the broker, not to the handshake. Until it exists, 8883 admits every+ * client exactly as 1883 does.+ */+ static final X509ExtendedTrustManager ACCEPT_ANY_CLIENT_CERTIFICATE = new X509ExtendedTrustManager() {+ @Override+ public void checkClientTrusted(X509Certificate[] chain, String authType) {+ }++ @Override+ public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) {+ }++ @Override+ public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine engine) {+ }++ @Override+ public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {+ throw new CertificateException("server-side trust manager");+ }++ @Override+ public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException {+ throw new CertificateException("server-side trust manager");+ }++ @Override+ public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine engine) throws CertificateException {+ throw new CertificateException("server-side trust manager");+ }++ @Override+ public X509Certificate[] getAcceptedIssuers() {+ return new X509Certificate[0];+ }+ };+private final EmulatorConfig config;private final Vertx vertx;private final Instance<IotService> iotService;+ private final TlsConfigurationRegistry tlsRegistry;private final Map<String, ClientSession> sessionsByClient = new ConcurrentHashMap<>();private final Map<String, Map<String, Subscription>> subscriptionsByClient = new ConcurrentHashMap<>();private MqttServer server;+ private MqttServer tlsServer;+ private ReloadingKeyManager keyManager;@Inject- public IotMqttBrokerService(EmulatorConfig config, Vertx vertx, Instance<IotService> iotService) {+ public IotMqttBrokerService(EmulatorConfig config, Vertx vertx, Instance<IotService> iotService,+ TlsConfigurationRegistry tlsRegistry) {this.config = config;this.vertx = vertx;this.iotService = iotService;+ this.tlsRegistry = tlsRegistry;}void onStart(@Observes StartupEvent ignored) {@@ -72,33 +129,100 @@ public class IotMqttBrokerService {return;}- MqttServer mqttServer = MqttServer.create(vertx, new MqttServerOptions()+ MqttServer mqttServer = listen(new MqttServerOptions().setHost(config.services().iot().mqtt().host())- .setPort(config.services().iot().mqtt().port()));- mqttServer.endpointHandler(this::handleEndpoint);- mqttServer.exceptionHandler(error -> LOG.warnv("IoT MQTT broker error: {0}", error.getMessage()));+ .setPort(config.services().iot().mqtt().port()), "IoT MQTT broker");+ try {+ startTlsListener();+ } catch (RuntimeException e) {+ mqttServer.close().toCompletionStage().toCompletableFuture().join();+ throw e;+ }+ server = mqttServer;+ }+ /**+ * The MQTT over TLS listener, AWS IoT's 8883, sharing every handler with the plaintext one.+ * Its key material is the TLS registry's default configuration, the same certificate the HTTPS+ * endpoint presents. Not opened when the port is 0 or TLS is off.+ */+ private void startTlsListener() {+ int tlsPort = config.services().iot().mqtt().tlsPort();+ if (tlsPort <= 0) {+ return;+ }+ if (!config.tls().enabled()) {+ LOG.infov("IoT MQTT TLS listener not started on port {0}: floci.tls.enabled is false", Integer.toString(tlsPort));+ return;+ }+ TlsConfiguration tls = tlsRegistry.getDefault().orElse(null);+ if (tls == null) {+ LOG.warnv("IoT MQTT TLS listener not started on port {0}: no default TLS configuration is registered",+ Integer.toString(tlsPort));+ return;+ }+ ReloadingKeyManager manager = new ReloadingKeyManager(ReloadingKeyManager.keyManagerOf(vertx, tls.getKeyStoreOptions()));+ tlsServer = listen(new MqttServerOptions()+ .setHost(config.services().iot().mqtt().host())+ .setPort(tlsPort)+ .setSsl(true)+ .setKeyCertOptions(KeyCertOptions.wrap(manager))+ .setTrustOptions(TrustOptions.wrap(ACCEPT_ANY_CLIENT_CERTIFICATE))+ .setClientAuth(ClientAuth.REQUEST), "IoT MQTT TLS broker");+ keyManager = manager;+ }++ private MqttServer listen(MqttServerOptions options, String name) {+ MqttServer mqttServer = MqttServer.create(vertx, options);+ mqttServer.endpointHandler(this::handleEndpoint);+ mqttServer.exceptionHandler(error -> LOG.warnv("{0} error: {1}", name, error.getMessage()));try {mqttServer.listen().toCompletionStage().toCompletableFuture().join();- server = mqttServer;- LOG.infov("IoT MQTT broker started on {0}:{1}",- config.services().iot().mqtt().host(), config.services().iot().mqtt().port());} catch (Exception e) {- mqttServer.close();- throw new IllegalStateException("Failed to start IoT MQTT broker", e);+ mqttServer.close().toCompletionStage().toCompletableFuture().join();+ throw new IllegalStateException("Failed to start " + name + " on port " + options.getPort(), e);+ }+ LOG.infov("{0} started on {1}:{2}", name, options.getHost(), Integer.toString(options.getPort()));+ return mqttServer;+ }++ /**+ * The HTTPS server switches to a reissued server certificate through this event (a hostname+ * learned by {@code TlsCertificateManager.ensureHost}, or a reload). The TLS listener follows+ * by handing its key manager the reloaded key store: the next handshake presents the new+ * certificate, established sessions keep theirs, and a failed reload keeps the previous one.+ */+ synchronized void onCertificateUpdated(@Observes CertificateUpdatedEvent event) {+ ReloadingKeyManager manager = keyManager;+ if (manager == null || !TlsConfig.DEFAULT_NAME.equalsIgnoreCase(event.name())) {+ return;+ }+ try {+ manager.reload(ReloadingKeyManager.keyManagerOf(vertx, event.tlsConfiguration().getKeyStoreOptions()));+ LOG.debug("IoT MQTT TLS broker serves the reloaded server certificate");+ } catch (Exception e) {+ LOG.warnv(e, "IoT MQTT TLS broker keeps its previous server certificate: {0}", e.getMessage());}}synchronized void stop() {MqttServer mqttServer = server;- if (mqttServer == null) {+ MqttServer mqttTlsServer = tlsServer;+ if (mqttServer == null && mqttTlsServer == null) {return;}server = null;+ tlsServer = null;+ keyManager = null;sessionsByClient.values().forEach(session -> session.endpoint().close());sessionsByClient.clear();subscriptionsByClient.clear();- mqttServer.close().toCompletionStage().toCompletableFuture().join();+ if (mqttServer != null) {+ mqttServer.close().toCompletionStage().toCompletableFuture().join();+ }+ if (mqttTlsServer != null) {+ mqttTlsServer.close().toCompletionStage().toCompletableFuture().join();+ }LOG.info("IoT MQTT broker stopped");}diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/ReloadingKeyManager.java b/src/main/java/io/github/hectorvent/floci/services/iot/ReloadingKeyManager.javanew file mode 100644index 0000000000..26911a31cd--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/iot/ReloadingKeyManager.java@@ -0,0 +1,172 @@+package io.github.hectorvent.floci.services.iot;++import io.vertx.core.Vertx;+import io.vertx.core.net.KeyCertOptions;++import javax.net.ssl.KeyManager;+import javax.net.ssl.KeyManagerFactory;+import javax.net.ssl.SSLEngine;+import javax.net.ssl.X509ExtendedKeyManager;+import javax.net.ssl.X509KeyManager;+import java.net.Socket;+import java.security.Principal;+import java.security.PrivateKey;+import java.security.cert.X509Certificate;+import java.time.Duration;+import java.util.ArrayList;+import java.util.List;+import java.util.function.LongSupplier;++/**+ * Server key manager for the MQTT TLS listener that answers every handshake from the key material+ * it was last given, so a reissued server certificate is served without rebuilding the listener's+ * SSL context: Vert.x drops the connections it accepts while a context is being rebuilt, whereas+ * JSSE asks the key manager for an alias, then that alias's key and chain, on every handshake.+ *+ * <p>The alias handed to JSSE names the generation it came from, and a retired generation stays+ * readable for {@link #RETIRED_GENERATION_RETENTION} after its replacement, longer than any+ * handshake can live (Vert.x closes a connection whose handshake exceeds its 10 second timeout), so+ * a handshake that began before any number of reloads still reads a matching key and chain.+ * Retired generations are dropped on the next reload after that window.+ */+final class ReloadingKeyManager extends X509ExtendedKeyManager {++ static final Duration RETIRED_GENERATION_RETENTION = Duration.ofSeconds(60);++ private record Generation(long number, X509KeyManager delegate, long retiredAtNanos) {+ }++ private final LongSupplier nanoTime;+ private volatile Generation current;+ /** Superseded generations, newest first, each within the retention window at its last reload. */+ private volatile List<Generation> retired = List.of();++ ReloadingKeyManager(X509KeyManager delegate) {+ this(delegate, System::nanoTime);+ }++ ReloadingKeyManager(X509KeyManager delegate, LongSupplier nanoTime) {+ this.nanoTime = nanoTime;+ this.current = new Generation(0, delegate, 0);+ }++ /** The first X.509 key manager of {@code options}, the TLS registry's default key store. */+ static X509KeyManager keyManagerOf(Vertx vertx, KeyCertOptions options) {+ if (options == null) {+ throw new IllegalStateException("the default TLS configuration has no key store");+ }+ KeyManagerFactory factory;+ try {+ factory = options.getKeyManagerFactory(vertx);+ } catch (Exception e) {+ throw new IllegalStateException("the default TLS configuration's key store cannot be loaded: " + e.getMessage(), e);+ }+ if (factory != null) {+ for (KeyManager manager : factory.getKeyManagers()) {+ if (manager instanceof X509KeyManager x509) {+ return x509;+ }+ }+ }+ throw new IllegalStateException("the default TLS configuration has no X.509 key manager");+ }++ /** Serves {@code delegate} from the next handshake on; the generations before it stay readable for a while. */+ synchronized void reload(X509KeyManager delegate) {+ long now = nanoTime.getAsLong();+ List<Generation> kept = new ArrayList<>();+ kept.add(new Generation(current.number(), current.delegate(), now));+ for (Generation generation : retired) {+ if (now - generation.retiredAtNanos() < RETIRED_GENERATION_RETENTION.toNanos()) {+ kept.add(generation);+ }+ }+ retired = List.copyOf(kept);+ current = new Generation(current.number() + 1, delegate, 0);+ }++ int retiredGenerations() {+ return retired.size();+ }++ @Override+ public String chooseEngineServerAlias(String keyType, Principal[] issuers, SSLEngine engine) {+ Generation generation = current;+ return qualify(generation, generation.delegate() instanceof X509ExtendedKeyManager extended+ ? extended.chooseEngineServerAlias(keyType, issuers, engine)+ : generation.delegate().chooseServerAlias(keyType, issuers, null));+ }++ @Override+ public String chooseServerAlias(String keyType, Principal[] issuers, Socket socket) {+ Generation generation = current;+ return qualify(generation, generation.delegate().chooseServerAlias(keyType, issuers, socket));+ }++ @Override+ public String[] getServerAliases(String keyType, Principal[] issuers) {+ Generation generation = current;+ String[] aliases = generation.delegate().getServerAliases(keyType, issuers);+ if (aliases == null) {+ return new String[0];+ }+ String[] qualified = new String[aliases.length];+ for (int i = 0; i < aliases.length; i++) {+ qualified[i] = qualify(generation, aliases[i]);+ }+ return qualified;+ }++ @Override+ public X509Certificate[] getCertificateChain(String alias) {+ Generation generation = generationOf(alias);+ return generation == null ? null : generation.delegate().getCertificateChain(unqualified(alias));+ }++ @Override+ public PrivateKey getPrivateKey(String alias) {+ Generation generation = generationOf(alias);+ return generation == null ? null : generation.delegate().getPrivateKey(unqualified(alias));+ }++ @Override+ public String[] getClientAliases(String keyType, Principal[] issuers) {+ return new String[0];+ }++ @Override+ public String chooseClientAlias(String[] keyType, Principal[] issuers, Socket socket) {+ return null;+ }++ @Override+ public String chooseEngineClientAlias(String[] keyType, Principal[] issuers, SSLEngine engine) {+ return null;+ }++ private static String qualify(Generation generation, String alias) {+ return alias == null ? null : generation.number() + ":" + alias;+ }++ private static String unqualified(String alias) {+ return alias.substring(alias.indexOf(':') + 1);+ }++ private Generation generationOf(String alias) {+ int separator = alias == null ? -1 : alias.indexOf(':');+ if (separator <= 0) {+ return null;+ }+ String number = alias.substring(0, separator);+ Generation candidate = current;+ if (Long.toString(candidate.number()).equals(number)) {+ return candidate;+ }+ for (Generation generation : retired) {+ if (Long.toString(generation.number()).equals(number)) {+ return generation;+ }+ }+ return null;+ }+}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 0cd7c50535..ba66846041 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -591,6 +591,7 @@ floci:auto-start: falsehost: 0.0.0.0port: 1883+ tls-port: 8883iotdata:enabled: truecloudhsmv2:diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerServiceTest.javanew file mode 100644index 0000000000..04898b22b0--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerServiceTest.java@@ -0,0 +1,453 @@+package io.github.hectorvent.floci.services.iot;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.config.FlociCertificateAuthority;+import io.github.hectorvent.floci.services.acm.CertificateGenerator;+import io.github.hectorvent.floci.services.acm.model.KeyAlgorithm;+import io.quarkus.tls.CertificateUpdatedEvent;+import io.quarkus.tls.TlsConfiguration;+import io.quarkus.tls.TlsConfigurationRegistry;+import io.quarkus.tls.runtime.config.TlsConfig;+import io.vertx.core.Vertx;+import io.vertx.core.buffer.Buffer;+import io.vertx.core.net.KeyCertOptions;+import io.vertx.core.net.PemKeyCertOptions;+import jakarta.enterprise.inject.Instance;+import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import javax.net.ssl.KeyManager;+import javax.net.ssl.KeyManagerFactory;+import javax.net.ssl.SSLContext;+import javax.net.ssl.SSLEngine;+import javax.net.ssl.SSLSession;+import javax.net.ssl.SSLSocket;+import javax.net.ssl.TrustManagerFactory;+import javax.net.ssl.X509ExtendedKeyManager;+import java.io.IOException;+import java.net.InetSocketAddress;+import java.net.ServerSocket;+import java.net.Socket;+import java.nio.file.Path;+import java.security.KeyStore;+import java.security.Principal;+import java.security.PrivateKey;+import java.security.cert.Certificate;+import java.security.cert.X509Certificate;+import java.util.ArrayList;+import java.util.List;+import java.util.Optional;+import java.util.Set;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verifyNoInteractions;+import static org.mockito.Mockito.when;++/**+ * The MQTT over TLS listener against a real Vert.x and a mocked TLS registry: what it serves,+ * when it is not opened, how a bind failure is reported, and how a reloaded server certificate+ * reaches the next handshake without a restart.+ */+class IotMqttBrokerServiceTest {++ private static final CertificateGenerator GENERATOR = new CertificateGenerator();++ /**+ * AWS IoT's default security policy, IoTSecurityPolicy_TLS13_1_2_2022_10, in JSSE names: the+ * TLS 1.3 suites and the RSA-authenticated TLS 1.2 suites it lists (transport-security page).+ */+ private static final Set<String> AWS_TLS13_SUITES = Set.of(+ "TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384", "TLS_CHACHA20_POLY1305_SHA256");+ private static final Set<String> AWS_TLS12_RSA_SUITES = Set.of(+ "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256",+ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",+ "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384", "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",+ "TLS_RSA_WITH_AES_128_GCM_SHA256", "TLS_RSA_WITH_AES_128_CBC_SHA256", "TLS_RSA_WITH_AES_128_CBC_SHA",+ "TLS_RSA_WITH_AES_256_GCM_SHA384", "TLS_RSA_WITH_AES_256_CBC_SHA256", "TLS_RSA_WITH_AES_256_CBC_SHA");++ private static Vertx vertx;+ private static FlociCertificateAuthority ca;+ private static CertificateGenerator.GeneratedCertificate bootLeaf;+ private static CertificateGenerator.GeneratedCertificate reissuedLeaf;++ private final EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ private final TlsConfigurationRegistry registry = mock(TlsConfigurationRegistry.class);+ private final TlsConfiguration tls = mock(TlsConfiguration.class);+ @SuppressWarnings("unchecked")+ private final Instance<IotService> iotService = mock(Instance.class);+ private int plainPort;+ private int tlsPort;+ private IotMqttBrokerService broker;++ @BeforeAll+ static void keyMaterial(@TempDir Path dir) {+ vertx = Vertx.vertx();+ ca = FlociCertificateAuthority.loadOrCreate(dir);+ bootLeaf = ca.issueServerCertificate("localhost", List.of("localhost", "127.0.0.1"), KeyAlgorithm.RSA_2048, null);+ reissuedLeaf = ca.issueServerCertificate("localhost",+ List.of("localhost", "127.0.0.1", "iot.dev.localhost.floci.io"), KeyAlgorithm.RSA_2048, null);+ }++ @AfterAll+ static void closeVertx() {+ vertx.close().toCompletionStage().toCompletableFuture().join();+ }++ @BeforeEach+ void brokerWithTlsOn() throws IOException {+ plainPort = freePort();+ do {+ tlsPort = freePort();+ } while (tlsPort == plainPort);+ when(config.services().iot().enabled()).thenReturn(true);+ when(config.services().iot().mqtt().enabled()).thenReturn(true);+ when(config.services().iot().mqtt().host()).thenReturn("127.0.0.1");+ when(config.services().iot().mqtt().port()).thenReturn(plainPort);+ when(config.services().iot().mqtt().tlsPort()).thenReturn(tlsPort);+ when(config.tls().enabled()).thenReturn(true);+ when(registry.getDefault()).thenReturn(Optional.of(tls));+ when(tls.getKeyStoreOptions()).thenReturn(pem(bootLeaf));+ broker = new IotMqttBrokerService(config, vertx, iotService, registry);+ }++ @AfterEach+ void stopBroker() {+ broker.stop();+ }++ @Test+ void tlsListenerServesTheRegistryLeafAndAsksForAClientCertificateWithoutRequiringOne() throws Exception {+ broker.startIfEnabled();++ RecordingKeyManager clientKeys = new RecordingKeyManager();+ X509Certificate served = handshake(new KeyManager[] {clientKeys});++ assertEquals(serial(bootLeaf), served.getSerialNumber(), "the leaf from the TLS registry's default configuration");+ assertTrue(clientKeys.asked.get(), "the server sent a CertificateRequest");+ assertTrue(accepts(plainPort), "the plaintext listener is up as well");+ }++ @Test+ void aDeviceOnAwsDefaultSecurityPolicyNegotiatesTls13() throws Exception {+ broker.startIfEnabled();++ SSLSession session = negotiate("TLSv1.3", AWS_TLS13_SUITES);++ assertEquals("TLSv1.3", session.getProtocol());+ assertTrue(AWS_TLS13_SUITES.contains(session.getCipherSuite()), session.getCipherSuite());+ }++ @Test+ void aDeviceOnAwsDefaultSecurityPolicyNegotiatesTls12() throws Exception {+ broker.startIfEnabled();++ SSLSession session = negotiate("TLSv1.2", AWS_TLS12_RSA_SUITES);++ assertEquals("TLSv1.2", session.getProtocol());+ assertTrue(AWS_TLS12_RSA_SUITES.contains(session.getCipherSuite()), session.getCipherSuite());+ }++ @Test+ void aClientCertificateFromAnyIssuerCompletesTheHandshake() throws Exception {+ broker.startIfEnabled();+ var device = GENERATOR.generateSelfSignedCertificate("device", List.of(), KeyAlgorithm.RSA_2048);++ X509Certificate served = handshake(keyManagers(device));++ assertEquals(serial(bootLeaf), served.getSerialNumber());+ }++ @Test+ void tlsPortZeroOpensOnlyThePlaintextListener() {+ when(config.services().iot().mqtt().tlsPort()).thenReturn(0);++ broker.startIfEnabled();++ assertTrue(accepts(plainPort));+ verifyNoInteractions(registry);+ }++ @Test+ void tlsDisabledOpensOnlyThePlaintextListener() {+ when(config.tls().enabled()).thenReturn(false);++ broker.startIfEnabled();++ assertTrue(accepts(plainPort));+ assertFalse(accepts(tlsPort), "no TLS listener while floci.tls.enabled is false");+ verifyNoInteractions(registry);+ }++ @Test+ void noDefaultTlsConfigurationLeavesTheTlsPortClosed() {+ when(registry.getDefault()).thenReturn(Optional.empty());++ broker.startIfEnabled();++ assertTrue(broker.isRunning());+ assertTrue(accepts(plainPort));+ assertFalse(accepts(tlsPort));+ }++ @Test+ void aTlsBindFailureLeavesNoListenerBehindAndTheNextStartSucceeds() throws Exception {+ try (ServerSocket blocker = new ServerSocket()) {+ blocker.bind(new InetSocketAddress("127.0.0.1", tlsPort));++ IllegalStateException failure = assertThrows(IllegalStateException.class, broker::startIfEnabled);++ assertTrue(failure.getMessage().contains(Integer.toString(tlsPort)), failure.getMessage());+ assertFalse(broker.isRunning());+ awaitClosed(plainPort);+ }++ broker.startIfEnabled();++ assertTrue(broker.isRunning());+ assertTrue(accepts(plainPort));+ assertEquals(serial(bootLeaf), handshake(null).getSerialNumber());+ }++ @Test+ void certificateUpdatedEventSwapsTheLeafForTheNextHandshake() throws Exception {+ broker.startIfEnabled();+ assertEquals(serial(bootLeaf), handshake(null).getSerialNumber());+ when(tls.getKeyStoreOptions()).thenReturn(pem(reissuedLeaf));++ broker.onCertificateUpdated(new CertificateUpdatedEvent(TlsConfig.DEFAULT_NAME, tls));++ assertEquals(serial(reissuedLeaf), handshake(null).getSerialNumber());+ }++ @Test+ void anEventForAnotherTlsConfigurationIsIgnored() throws Exception {+ broker.startIfEnabled();+ when(tls.getKeyStoreOptions()).thenReturn(pem(reissuedLeaf));++ broker.onCertificateUpdated(new CertificateUpdatedEvent("rds-proxy", tls));++ assertEquals(serial(bootLeaf), handshake(null).getSerialNumber());+ }++ @Test+ void aFailedSwapKeepsThePreviousLeaf() throws Exception {+ broker.startIfEnabled();+ when(tls.getKeyStoreOptions()).thenReturn(new PemKeyCertOptions()+ .addCertValue(Buffer.buffer("not a certificate"))+ .addKeyValue(Buffer.buffer("not a key")));++ broker.onCertificateUpdated(new CertificateUpdatedEvent(TlsConfig.DEFAULT_NAME, tls));++ assertEquals(serial(bootLeaf), handshake(null).getSerialNumber());+ }++ @Test+ void eventsBeforeStartAndAfterStopAreIgnored() {+ broker.onCertificateUpdated(new CertificateUpdatedEvent(TlsConfig.DEFAULT_NAME, tls));++ broker.startIfEnabled();+ broker.stop();+ broker.onCertificateUpdated(new CertificateUpdatedEvent(TlsConfig.DEFAULT_NAME, tls));++ assertFalse(broker.isRunning());+ awaitClosed(tlsPort);+ awaitClosed(plainPort);+ }++ @Test+ void stopClosesBothListenersAndStartReopensThem() throws Exception {+ broker.startIfEnabled();+ broker.stop();+ awaitClosed(tlsPort);+ awaitClosed(plainPort);+ when(tls.getKeyStoreOptions()).thenReturn(pem(reissuedLeaf));++ broker.startIfEnabled();++ assertTrue(accepts(plainPort));+ assertEquals(serial(reissuedLeaf), handshake(null).getSerialNumber(), "a restart reads the registry again");+ }++ /**+ * Reloads while connections arrive: every handshake completes with one of the two leaves.+ * Rebuilding the listener's SSL context instead would drop the connections accepted while+ * the new context is being built (Vert.x serves a pending update to them as null).+ */+ @Test+ void concurrentCertificateEventsAndHandshakesAllSucceed() throws Exception {+ broker.startIfEnabled();+ when(tls.getKeyStoreOptions()).thenAnswer(ignored -> pem(reissuedLeaf));+ int threads = 8;+ int rounds = 10;+ CountDownLatch start = new CountDownLatch(1);+ ExecutorService pool = Executors.newFixedThreadPool(threads * 2);+ List<Future<?>> outcomes = new ArrayList<>();+ try {+ for (int i = 0; i < threads; i++) {+ outcomes.add(pool.submit(() -> {+ start.await();+ for (int round = 0; round < rounds; round++) {+ broker.onCertificateUpdated(new CertificateUpdatedEvent(TlsConfig.DEFAULT_NAME, tls));+ }+ return null;+ }));+ outcomes.add(pool.submit(() -> {+ start.await();+ for (int round = 0; round < rounds; round++) {+ X509Certificate served = handshake(null);+ assertTrue(served.getSerialNumber().equals(serial(bootLeaf))+ || served.getSerialNumber().equals(serial(reissuedLeaf)), "one of the two leaves");+ }+ return null;+ }));+ }+ start.countDown();+ for (Future<?> outcome : outcomes) {+ outcome.get(30, TimeUnit.SECONDS);+ }+ } finally {+ pool.shutdownNow();+ }++ assertEquals(serial(reissuedLeaf), handshake(null).getSerialNumber());+ }++ private X509Certificate handshake(KeyManager[] clientKeys) throws Exception {+ try (SSLSocket socket = connectTls(clientKeys)) {+ socket.startHandshake();+ return (X509Certificate) socket.getSession().getPeerCertificates()[0];+ }+ }++ /** A handshake as a device pinned to one protocol version and AWS's suites for it would do. */+ private SSLSession negotiate(String protocol, Set<String> suites) throws Exception {+ try (SSLSocket socket = connectTls(null)) {+ socket.setEnabledProtocols(new String[] {protocol});+ socket.setEnabledCipherSuites(List.of(socket.getSupportedCipherSuites()).stream()+ .filter(suites::contains).toArray(String[]::new));+ socket.startHandshake();+ return socket.getSession();+ }+ }++ private SSLSocket connectTls(KeyManager[] clientKeys) throws Exception {+ KeyStore trust = KeyStore.getInstance(KeyStore.getDefaultType());+ trust.load(null, null);+ trust.setCertificateEntry("floci", ca.certificate());+ TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());+ tmf.init(trust);+ SSLContext context = SSLContext.getInstance("TLS");+ context.init(clientKeys, tmf.getTrustManagers(), null);+ SSLSocket socket = (SSLSocket) context.getSocketFactory().createSocket();+ socket.connect(new InetSocketAddress("127.0.0.1", tlsPort), 5_000);+ return socket;+ }++ private static KeyManager[] keyManagers(CertificateGenerator.GeneratedCertificate leaf) throws Exception {+ KeyStore keys = KeyStore.getInstance(KeyStore.getDefaultType());+ keys.load(null, null);+ PrivateKey key = GENERATOR.parsePrivateKey(leaf.privateKeyPem());+ keys.setKeyEntry("device", key, new char[0], new Certificate[] {GENERATOR.parseCertificate(leaf.certificatePem())});+ KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());+ kmf.init(keys, new char[0]);+ return kmf.getKeyManagers();+ }++ private static KeyCertOptions pem(CertificateGenerator.GeneratedCertificate leaf) {+ return new PemKeyCertOptions()+ .addCertValue(Buffer.buffer(leaf.certificatePem()))+ .addKeyValue(Buffer.buffer(leaf.privateKeyPem()));+ }++ private static java.math.BigInteger serial(CertificateGenerator.GeneratedCertificate leaf) {+ return GENERATOR.parseCertificate(leaf.certificatePem()).getSerialNumber();+ }++ private static boolean accepts(int port) {+ try (Socket socket = new Socket()) {+ socket.connect(new InetSocketAddress("127.0.0.1", port), 500);+ return true;+ } catch (IOException refused) {+ return false;+ }+ }++ /** Vert.x resolves close() a moment before the OS releases the port; poll like the lazy-start test. */+ private static void awaitClosed(int port) {+ long deadline = System.currentTimeMillis() + 5_000;+ while (accepts(port)) {+ if (System.currentTimeMillis() > deadline) {+ throw new AssertionError("port " + port + " still accepts connections");+ }+ try {+ Thread.sleep(50);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AssertionError("interrupted while waiting for port " + port + " to close", e);+ }+ }+ }++ private static int freePort() throws IOException {+ try (ServerSocket socket = new ServerSocket(0)) {+ return socket.getLocalPort();+ }+ }++ /** A client key manager with no certificate that records whether the server asked for one. */+ private static final class RecordingKeyManager extends X509ExtendedKeyManager {+ final AtomicBoolean asked = new AtomicBoolean();++ @Override+ public String chooseClientAlias(String[] keyType, Principal[] issuers, Socket socket) {+ asked.set(true);+ return null;+ }++ @Override+ public String chooseEngineClientAlias(String[] keyType, Principal[] issuers, SSLEngine engine) {+ asked.set(true);+ return null;+ }++ @Override+ public String[] getClientAliases(String keyType, Principal[] issuers) {+ return new String[0];+ }++ @Override+ public String[] getServerAliases(String keyType, Principal[] issuers) {+ return new String[0];+ }++ @Override+ public String chooseServerAlias(String keyType, Principal[] issuers, Socket socket) {+ return null;+ }++ @Override+ public X509Certificate[] getCertificateChain(String alias) {+ return null;+ }++ @Override+ public PrivateKey getPrivateKey(String alias) {+ return null;+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttTlsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttTlsIntegrationTest.javanew file mode 100644index 0000000000..86d132079d--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttTlsIntegrationTest.java@@ -0,0 +1,371 @@+package io.github.hectorvent.floci.services.iot;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.CertificateMetadata;+import io.github.hectorvent.floci.config.FlociCertificateAuthority;+import io.github.hectorvent.floci.config.TlsCertificateManager;+import io.github.hectorvent.floci.services.acm.CertificateGenerator;+import io.github.hectorvent.floci.services.acm.model.KeyAlgorithm;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;+import jakarta.inject.Inject;+import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;+import org.eclipse.paho.client.mqttv3.MqttCallback;+import org.eclipse.paho.client.mqttv3.MqttClient;+import org.eclipse.paho.client.mqttv3.MqttConnectOptions;+import org.eclipse.paho.client.mqttv3.MqttException;+import org.eclipse.paho.client.mqttv3.MqttMessage;+import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;+import org.junit.jupiter.api.Test;++import javax.net.ssl.KeyManager;+import javax.net.ssl.KeyManagerFactory;+import javax.net.ssl.SSLContext;+import javax.net.ssl.SSLHandshakeException;+import javax.net.ssl.SSLParameters;+import javax.net.ssl.SSLSocket;+import javax.net.ssl.TrustManagerFactory;+import java.io.IOException;+import java.net.Socket;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.security.KeyStore;+import java.security.cert.Certificate;+import java.security.cert.X509Certificate;+import java.time.Duration;+import java.time.Instant;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.TreeSet;+import java.util.concurrent.BlockingQueue;+import java.util.concurrent.LinkedBlockingQueue;+import java.util.concurrent.TimeUnit;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.junit.jupiter.api.Assertions.assertArrayEquals;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * Devices connect to the MQTT over TLS listener the way they connect to AWS IoT's 8883: over+ * {@code ssl://}, trusting one CA, verifying the endpoint name, with MQTT 3.1.1 or MQTT 5, and+ * with the same broker behaviour as the plaintext listener.+ */+@QuarkusTest+@TestProfile(IotMqttTlsIntegrationTest.Profile.class)+class IotMqttTlsIntegrationTest {++ static final Path DATA_DIR = Path.of("target", "floci-iot-mqtt-tls-test").toAbsolutePath();+ static final Path TLS_DIR = DATA_DIR.resolve("tls");+ static final int PLAIN_PORT = 18834;+ static final int TLS_PORT = 18835;+ static final String LEARNED_HOST = "iot.dev.localhost.floci.io";+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();++ @Inject+ TlsCertificateManager certificateManager;++ @Inject+ IotMqttBrokerService broker;++ @Test+ void connectSubscribePublishOverTlsTrustingOnlyTheFlociCa() throws Exception {+ String topic = "tls/roundtrip/" + System.nanoTime();+ byte[] payload = "hello over 8883".getBytes(StandardCharsets.UTF_8);++ try (TlsClient subscriber = TlsClient.connect("tls-sub-" + System.nanoTime(), null)) {+ subscriber.subscribe(topic, 1);+ try (TlsClient publisher = TlsClient.connect("tls-pub-" + System.nanoTime(), null)) {+ publisher.publish(topic, payload, 1);+ }+ assertArrayEquals(payload, subscriber.takePayload());+ }+ }++ @Test+ void mqtt5ConnectsOverTls() throws Exception {+ org.eclipse.paho.mqttv5.client.MqttClient client = new org.eclipse.paho.mqttv5.client.MqttClient(+ "ssl://127.0.0.1:" + TLS_PORT, "tls-v5-" + System.nanoTime(), null);+ org.eclipse.paho.mqttv5.client.MqttConnectionOptions options =+ new org.eclipse.paho.mqttv5.client.MqttConnectionOptions();+ options.setCleanStart(true);+ options.setConnectionTimeout(10);+ options.setSocketFactory(trustOnlyTheCa(null).getSocketFactory());+ try {+ client.connect(options);+ assertTrue(client.isConnected());+ client.disconnect();+ } finally {+ client.close();+ }+ }++ @Test+ void aDeviceCertificateIssuedByTheCaIsAcceptedAndTheSessionIsVisibleToTheConnectionApi() throws Exception {+ String clientId = "tls-device-" + System.nanoTime();+ var device = FlociCertificateAuthority.loadOrCreate(TLS_DIR).issueClientCertificate("device-" + clientId);++ try (TlsClient client = TlsClient.connect(clientId, keyManagers(device))) {+ assertTrue(client.isConnected());+ given()+ .queryParam("includeSocketInformation", true)+ .when()+ .get("/connections/{clientId}", clientId)+ .then()+ .statusCode(200)+ .body("connected", equalTo(true))+ .body("sourceIp", equalTo("127.0.0.1"));+ }+ }++ @Test+ void plaintextOnTheTlsPortIsRefused() throws Exception {+ MqttClient plain = new MqttClient("tcp://127.0.0.1:" + TLS_PORT, "plain-on-tls-" + System.nanoTime(), new MemoryPersistence());+ try {+ MqttConnectOptions options = new MqttConnectOptions();+ options.setConnectionTimeout(5);++ assertThrows(MqttException.class, () -> plain.connect(options));++ assertFalse(plain.isConnected());+ } finally {+ plain.close();+ }+ }++ @Test+ void plaintextListenerStillWorks() throws Exception {+ MqttClient client = new MqttClient("tcp://127.0.0.1:" + PLAIN_PORT, "plain-" + System.nanoTime(), new MemoryPersistence());+ try {+ client.connect();+ assertTrue(client.isConnected());+ client.disconnect();+ } finally {+ client.close();+ }+ }++ @Test+ void learnedHostnameIsPresentedOnTheNextTlsHandshake() throws Exception {+ assertThrows(SSLHandshakeException.class, () -> handshakeVerifying(LEARNED_HOST),+ "precondition: a device verifying the custom domain rejects the boot leaf");++ certificateManager.ensureHost(LEARNED_HOST);++ Set<String> sans = sans(handshakeVerifying(LEARNED_HOST));+ assertTrue(sans.contains(LEARNED_HOST), sans.toString());+ assertTrue(sans.contains("localhost"), "configured names are still served: " + sans);+ }++ @Test+ void aClientIdReconnectingOnThePlaintextListenerTakesOverTheTlsSession() throws Exception {+ String clientId = "tls-takeover-" + System.nanoTime();++ try (TlsClient first = TlsClient.connect(clientId, null)) {+ MqttClient second = new MqttClient("tcp://127.0.0.1:" + PLAIN_PORT, clientId, new MemoryPersistence());+ try {+ second.connect();+ first.awaitDisconnected();+ assertTrue(second.isConnected());+ second.disconnect();+ } finally {+ second.close();+ }+ }+ }++ @Test+ void shadowUpdateOverTlsPublishesTheAcceptedResponseOverTls() throws Exception {+ String thing = "tlsThing" + System.nanoTime();+ try (TlsClient subscriber = TlsClient.connect("tls-shadow-sub-" + System.nanoTime(), null)) {+ subscriber.subscribe("$aws/things/" + thing + "/shadow/update/accepted", 0);+ try (TlsClient publisher = TlsClient.connect("tls-shadow-pub-" + System.nanoTime(), null)) {+ publisher.publish("$aws/things/" + thing + "/shadow/update",+ "{\"state\":{\"desired\":{\"color\":\"blue\"}},\"clientToken\":\"tls-token\"}".getBytes(StandardCharsets.UTF_8), 0);+ }+ String accepted = new String(subscriber.takePayload(), StandardCharsets.UTF_8);+ assertEquals("blue", OBJECT_MAPPER.readTree(accepted).path("state").path("desired").path("color").asText());+ assertEquals("tls-token", OBJECT_MAPPER.readTree(accepted).path("clientToken").asText());+ }+ }++ @Test+ void aRestartOfTheBrokerServesTlsAgain() throws Exception {+ broker.stop();+ broker.startIfEnabled();++ try (TlsClient client = TlsClient.connect("tls-restart-" + System.nanoTime(), null)) {+ assertTrue(client.isConnected());+ }+ }++ /**+ * Connects by IP and hands JSSE the custom domain as the peer host, which is what a device SDK+ * does once DNS points the name at Floci: SNI carries the name and the certificate must verify+ * for it. The boot leaf's wildcard covers one label, so the three-label name is uncovered.+ */+ private static X509Certificate handshakeVerifying(String host) throws Exception {+ try (Socket raw = new Socket("127.0.0.1", TLS_PORT);+ SSLSocket socket = (SSLSocket) trustOnlyTheCa(null).getSocketFactory().createSocket(raw, host, TLS_PORT, true)) {+ SSLParameters params = socket.getSSLParameters();+ params.setEndpointIdentificationAlgorithm("HTTPS");+ socket.setSSLParameters(params);+ socket.startHandshake();+ return (X509Certificate) socket.getSession().getPeerCertificates()[0];+ }+ }++ private static SSLContext trustOnlyTheCa(KeyManager[] clientKeys) throws Exception {+ KeyStore trust = KeyStore.getInstance(KeyStore.getDefaultType());+ trust.load(null, null);+ trust.setCertificateEntry("floci", FlociCertificateAuthority.loadOrCreate(TLS_DIR).certificate());+ TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());+ tmf.init(trust);+ SSLContext context = SSLContext.getInstance("TLS");+ context.init(clientKeys, tmf.getTrustManagers(), null);+ return context;+ }++ private static KeyManager[] keyManagers(CertificateGenerator.GeneratedCertificate leaf) throws Exception {+ CertificateGenerator generator = new CertificateGenerator();+ KeyStore keys = KeyStore.getInstance(KeyStore.getDefaultType());+ keys.load(null, null);+ keys.setKeyEntry("device", generator.parsePrivateKey(leaf.privateKeyPem()), new char[0],+ new Certificate[] {generator.parseCertificate(leaf.certificatePem())});+ KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());+ kmf.init(keys, new char[0]);+ return kmf.getKeyManagers();+ }++ private static Set<String> sans(X509Certificate leaf) throws Exception {+ Set<String> sans = new TreeSet<>();+ for (List<?> entry : leaf.getSubjectAlternativeNames()) {+ sans.add(String.valueOf(entry.get(1)));+ }+ return sans;+ }++ /** A Paho MQTT 3.1.1 client on {@code ssl://}, with Paho's default hostname verification on. */+ private static final class TlsClient implements AutoCloseable {+ private final MqttClient client;+ private final BlockingQueue<byte[]> payloads = new LinkedBlockingQueue<>();++ private TlsClient(MqttClient client) {+ this.client = client;+ }++ static TlsClient connect(String clientId, KeyManager[] clientKeys) throws Exception {+ MqttClient client = new MqttClient("ssl://127.0.0.1:" + TLS_PORT, clientId, new MemoryPersistence());+ TlsClient tlsClient = new TlsClient(client);+ client.setCallback(new MqttCallback() {+ @Override+ public void connectionLost(Throwable cause) {+ }++ @Override+ public void messageArrived(String topic, MqttMessage message) {+ tlsClient.payloads.add(message.getPayload());+ }++ @Override+ public void deliveryComplete(IMqttDeliveryToken token) {+ }+ });+ MqttConnectOptions options = new MqttConnectOptions();+ options.setCleanSession(true);+ options.setConnectionTimeout(10);+ options.setAutomaticReconnect(false);+ options.setSocketFactory(trustOnlyTheCa(clientKeys).getSocketFactory());+ client.connect(options);+ return tlsClient;+ }++ boolean isConnected() {+ return client.isConnected();+ }++ void subscribe(String topic, int qos) throws MqttException {+ client.subscribe(topic, qos);+ }++ void publish(String topic, byte[] payload, int qos) throws MqttException {+ client.publish(topic, payload, qos, false);+ }++ byte[] takePayload() throws InterruptedException {+ byte[] payload = payloads.poll(10, TimeUnit.SECONDS);+ assertNotNull(payload, "no publish received over TLS within 10s");+ return payload;+ }++ void awaitDisconnected() throws InterruptedException {+ Instant deadline = Instant.now().plus(Duration.ofSeconds(5));+ while (Instant.now().isBefore(deadline)) {+ if (!client.isConnected()) {+ return;+ }+ Thread.sleep(25);+ }+ throw new AssertionError("the TLS session stayed open after its client id reconnected elsewhere");+ }++ @Override+ public void close() throws MqttException {+ if (client.isConnected()) {+ client.disconnect();+ }+ client.close();+ }+ }++ /**+ * TLS on with a CA-issued leaf on disk, as TlsConfigSource leaves it in production. The+ * bootstrap reads system properties, not profile overrides, so the profile lays the files+ * down itself and points the TLS registry's default entry at them. The leaf carries+ * {@code 127.0.0.1} so Paho's hostname verification passes against {@code ssl://127.0.0.1}.+ */+ public static final class Profile implements QuarkusTestProfile {++ static {+ try {+ Files.createDirectories(TLS_DIR);+ for (String name : List.of("floci-server.crt", "floci-server.key", "floci-server.metadata.json")) {+ Files.deleteIfExists(TLS_DIR.resolve(name));+ }+ FlociCertificateAuthority ca = FlociCertificateAuthority.loadOrCreate(TLS_DIR);+ List<String> sans = List.of("localhost", "127.0.0.1", "*.localhost.floci.io");+ var leaf = ca.issueServerCertificate("localhost", sans, KeyAlgorithm.RSA_2048, null);+ Files.writeString(TLS_DIR.resolve("floci-server.crt"), leaf.certificatePem());+ Files.writeString(TLS_DIR.resolve("floci-server.key"), leaf.privateKeyPem());+ Files.writeString(TLS_DIR.resolve("floci-server.metadata.json"),+ OBJECT_MAPPER.writeValueAsString(CertificateMetadata.create(sans, "dev")));+ } catch (IOException e) {+ throw new IllegalStateException("could not prepare the TLS fixtures", e);+ }+ }++ @Override+ public Map<String, String> getConfigOverrides() {+ return Map.ofEntries(+ Map.entry("floci.tls.enabled", "true"),+ Map.entry("floci.tls.self-signed", "true"),+ Map.entry("floci.tls.aws-https-port", "0"),+ Map.entry("floci.storage.persistent-path", DATA_DIR.toString()),+ Map.entry("quarkus.tls.key-store.pem.0.cert", TLS_DIR.resolve("floci-server.crt").toString()),+ Map.entry("quarkus.tls.key-store.pem.0.key", TLS_DIR.resolve("floci-server.key").toString()),+ Map.entry("quarkus.http.insecure-requests", "enabled"),+ Map.entry("floci.services.iot.mqtt.enabled", "true"),+ Map.entry("floci.services.iot.mqtt.auto-start", "true"),+ Map.entry("floci.services.iot.mqtt.host", "127.0.0.1"),+ Map.entry("floci.services.iot.mqtt.port", Integer.toString(PLAIN_PORT)),+ Map.entry("floci.services.iot.mqtt.tls-port", Integer.toString(TLS_PORT)));+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/ReloadingKeyManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/ReloadingKeyManagerTest.javanew file mode 100644index 0000000000..5220ba19e8--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/iot/ReloadingKeyManagerTest.java@@ -0,0 +1,189 @@+package io.github.hectorvent.floci.services.iot;++import io.github.hectorvent.floci.services.acm.CertificateGenerator;+import io.github.hectorvent.floci.services.acm.model.KeyAlgorithm;+import io.vertx.core.Vertx;+import io.vertx.core.buffer.Buffer;+import io.vertx.core.net.PemKeyCertOptions;+import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import javax.net.ssl.SSLEngine;+import javax.net.ssl.X509ExtendedKeyManager;+import java.security.PrivateKey;+import java.security.cert.X509Certificate;+import java.util.List;+import java.util.concurrent.atomic.AtomicLong;++import static org.junit.jupiter.api.Assertions.assertArrayEquals;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertSame;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++/**+ * JSSE asks a key manager for an alias, then for that alias's key and chain, on every handshake.+ * The manager under test answers from its current delegate and keeps a handshake that started+ * before a reload on the delegate it began with.+ */+class ReloadingKeyManagerTest {++ private static Vertx vertx;++ @BeforeAll+ static void startVertx() {+ vertx = Vertx.vertx();+ }++ @AfterAll+ static void closeVertx() {+ vertx.close().toCompletionStage().toCompletableFuture().join();+ }++ @Test+ void serverAliasAndItsMaterialComeFromTheDelegate() {+ X509ExtendedKeyManager inner = delegate("srv", "first");++ ReloadingKeyManager manager = new ReloadingKeyManager(inner);++ String alias = manager.chooseEngineServerAlias("RSA", null, null);+ assertNotNull(alias);+ assertSame(inner.getCertificateChain("srv"), manager.getCertificateChain(alias));+ assertSame(inner.getPrivateKey("srv"), manager.getPrivateKey(alias));+ assertEquals(alias, manager.chooseServerAlias("RSA", null, null), "socket and engine flavours agree");+ assertArrayEquals(new String[] {alias}, manager.getServerAliases("RSA", null));+ }++ @Test+ void reloadServesTheNewDelegateToTheNextHandshake() {+ X509ExtendedKeyManager first = delegate("srv", "first");+ X509ExtendedKeyManager second = delegate("srv", "second");+ ReloadingKeyManager manager = new ReloadingKeyManager(first);++ manager.reload(second);++ String alias = manager.chooseEngineServerAlias("RSA", null, null);+ assertSame(second.getCertificateChain("srv"), manager.getCertificateChain(alias));+ assertSame(second.getPrivateKey("srv"), manager.getPrivateKey(alias));+ }++ @Test+ void anAliasChosenBeforeAReloadKeepsItsOwnChainAndKey() {+ X509ExtendedKeyManager first = delegate("srv", "first");+ X509ExtendedKeyManager second = delegate("srv", "second");+ ReloadingKeyManager manager = new ReloadingKeyManager(first);+ String inFlight = manager.chooseEngineServerAlias("RSA", null, null);++ manager.reload(second);++ assertSame(first.getCertificateChain("srv"), manager.getCertificateChain(inFlight));+ assertSame(first.getPrivateKey("srv"), manager.getPrivateKey(inFlight));+ }++ @Test+ void anAliasChosenBeforeSeveralQuickReloadsStillResolvesItsOwnChainAndKey() {+ X509ExtendedKeyManager first = delegate("srv", "first");+ ReloadingKeyManager manager = new ReloadingKeyManager(first);+ String inFlight = manager.chooseEngineServerAlias("RSA", null, null);++ manager.reload(delegate("srv", "second"));+ manager.reload(delegate("srv", "third"));+ manager.reload(delegate("srv", "fourth"));++ assertSame(first.getCertificateChain("srv"), manager.getCertificateChain(inFlight));+ assertSame(first.getPrivateKey("srv"), manager.getPrivateKey(inFlight));+ }++ @Test+ void aRetiredGenerationIsDroppedOnceNoHandshakeCanStillReferenceIt() {+ AtomicLong clock = new AtomicLong();+ X509ExtendedKeyManager first = delegate("srv", "first");+ ReloadingKeyManager manager = new ReloadingKeyManager(first, clock::get);+ String stale = manager.chooseEngineServerAlias("RSA", null, null);+ manager.reload(delegate("srv", "second"));+ clock.addAndGet(ReloadingKeyManager.RETIRED_GENERATION_RETENTION.toNanos() - 1);+ manager.reload(delegate("srv", "third"));+ assertSame(first.getPrivateKey("srv"), manager.getPrivateKey(stale), "still inside the retention window");++ clock.addAndGet(1);+ manager.reload(delegate("srv", "fourth"));++ assertNull(manager.getCertificateChain(stale));+ assertNull(manager.getPrivateKey(stale));+ assertEquals(2, manager.retiredGenerations(), "the two generations retired inside the window are kept, the first is gone");+ }++ @Test+ void unknownAliasesAndKeyTypesYieldNull() {+ ReloadingKeyManager manager = new ReloadingKeyManager(delegate("srv", "first"));++ assertNull(manager.chooseEngineServerAlias("EC", null, null));+ assertNull(manager.chooseServerAlias("EC", null, null));+ assertEquals(0, manager.getServerAliases("EC", null).length);+ assertNull(manager.getCertificateChain("garbage"));+ assertNull(manager.getPrivateKey("garbage"));+ assertNull(manager.getCertificateChain("999:srv"));+ assertNull(manager.getPrivateKey(null));+ }++ @Test+ void neverActsAsATlsClient() {+ ReloadingKeyManager manager = new ReloadingKeyManager(delegate("srv", "first"));++ assertNull(manager.chooseClientAlias(new String[] {"RSA"}, null, null));+ assertNull(manager.chooseEngineClientAlias(new String[] {"RSA"}, null, null));+ assertEquals(0, manager.getClientAliases("RSA", null).length);+ }++ @Test+ void keyManagerOfLoadsThePemKeyStoreOfTheOptions() {+ var leaf = new CertificateGenerator().generateSelfSignedCertificate("localhost", List.of("localhost"), KeyAlgorithm.RSA_2048);+ PemKeyCertOptions options = new PemKeyCertOptions()+ .addCertValue(Buffer.buffer(leaf.certificatePem()))+ .addKeyValue(Buffer.buffer(leaf.privateKeyPem()));++ ReloadingKeyManager manager = new ReloadingKeyManager(ReloadingKeyManager.keyManagerOf(vertx, options));++ String alias = manager.chooseEngineServerAlias("RSA", null, null);+ assertNotNull(alias);+ assertEquals(new CertificateGenerator().parseCertificate(leaf.certificatePem()), manager.getCertificateChain(alias)[0]);+ assertNotNull(manager.getPrivateKey(alias));+ }++ @Test+ void keyManagerOfRefusesAConfigurationWithoutAKeyStore() {+ IllegalStateException refused = assertThrows(IllegalStateException.class,+ () -> ReloadingKeyManager.keyManagerOf(vertx, null));++ assertEquals("the default TLS configuration has no key store", refused.getMessage());+ }++ @Test+ void keyManagerOfReportsAKeyStoreThatCannotBeLoaded() {+ PemKeyCertOptions broken = new PemKeyCertOptions()+ .addCertValue(Buffer.buffer("not a certificate"))+ .addKeyValue(Buffer.buffer("not a key"));++ IllegalStateException refused = assertThrows(IllegalStateException.class,+ () -> ReloadingKeyManager.keyManagerOf(vertx, broken));++ assertNotNull(refused.getCause());+ }++ /** A delegate with one server alias for RSA whose chain and key are distinct mock objects. */+ private static X509ExtendedKeyManager delegate(String alias, String label) {+ X509ExtendedKeyManager inner = mock(X509ExtendedKeyManager.class, label);+ X509Certificate cert = mock(X509Certificate.class, label + "-cert");+ PrivateKey key = mock(PrivateKey.class, label + "-key");+ when(inner.chooseEngineServerAlias("RSA", null, null)).thenReturn(alias);+ when(inner.chooseServerAlias("RSA", null, null)).thenReturn(alias);+ when(inner.getServerAliases("RSA", null)).thenReturn(new String[] {alias});+ when(inner.getCertificateChain(alias)).thenReturn(new X509Certificate[] {cert});+ when(inner.getPrivateKey(alias)).thenReturn(key);+ return inner;+ }+}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 4ff6210d39..d565899684 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -354,6 +354,7 @@ floci:auto-start: falsehost: 127.0.0.1port: 1883+ tls-port: 8883iotdata:enabled: truecloudhsmv2: