Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(iot): MQTT over TLS listener on 8883 (#3142)

Mario Toffiaee2f55b78422
diff --git a/docs/configuration/tls.md b/docs/configuration/tls.md
index 970b9b0efa..7786beb727 100644
--- a/docs/configuration/tls.md
+++ b/docs/configuration/tls.md
@@ -123,6 +123,10 @@ wss://localhost:4566/ws/{apiId}/{stage}
No additional configuration is needed: Vert.x handles TLS at the transport layer transparently.
+## MQTT over TLS (8883)
+
+When TLS is enabled, the IoT MQTT broker also listens on port 8883 with the same certificate as the HTTPS endpoint, hostnames learned at runtime included. Devices connect with `ssl://` trusting `ca.pem`; see [IoT Core](../services/iot.md#mqtt-over-tls).
+
## SDK Configuration Examples
With `AWS_CA_BUNDLE` set as in Quick Start, no SDK needs code changes. The examples below trust the CA in code instead, for processes whose environment you do not control.
diff --git a/docs/services/iot.md b/docs/services/iot.md
index fce6772ba8..2ed9b5088f 100644
--- a/docs/services/iot.md
+++ b/docs/services/iot.md
@@ -79,11 +79,23 @@ Broker scope:
- Target real AWS IoT/device SDK style MQTT clients, not only handcrafted packet tests.
- Support MQTT v3 and MQTT 5 CONNECT handling used by local compatibility tests.
- Support QoS 0 and QoS 1 publish/subscribe behavior for the local AWS IoT slice.
-- Keep MQTT plaintext-only for this phase; TLS and mTLS are out of scope.
+- Serve MQTT over TLS on 8883 next to plaintext 1883 when TLS is enabled; verifying device certificates (mutual TLS) is follow-up scope.
- Keep MQTT authorization permissive for now, but leave room for a later pluggable IoT certificate and policy authorizer.
- Keep MQTT broker logging minimal.
- Validate the relevant IoT compatibility tests against the native binary before considering the phase complete.
+### MQTT over TLS
+
+With `FLOCI_TLS_ENABLED=true` the broker also listens on `FLOCI_SERVICES_IOT_MQTT_TLS_PORT` (default `8883`, the port AWS IoT uses for X.509 device connections; `0` disables it). It presents the same certificate as the HTTPS endpoint, issued by the Floci CA, over TLS 1.2 or 1.3. A device connects with `ssl://localhost:8883` trusting `GET /_floci/ca.pem`, as it would trust Amazon Root CA 1 against AWS IoT. A custom domain added to the server certificate at runtime (see [TLS](../configuration/tls.md#custom-domains-learned-at-runtime)) is served on 8883 from the next connection on, without a restart.
+
+The listener asks for a client certificate and accepts the connection whether or not one is presented and whoever signed it: 8883 is as permissive as 1883 until certificate verification lands. Sessions, subscriptions and reserved topics are shared with the plaintext listener, so a client id connecting on one port replaces its session on the other, as on AWS. Both listeners start together: with the first IoT API call, or at boot with `FLOCI_SERVICES_IOT_MQTT_AUTO_START=true`.
+
+```bash
+docker run -e FLOCI_TLS_ENABLED=true -e FLOCI_SERVICES_IOT_MQTT_AUTO_START=true -p 4566:4566 -p 8883:8883 floci/floci:latest
+curl http://localhost:4566/_floci/ca.pem -o ca.pem
+mosquitto_sub -h localhost -p 8883 --cafile ca.pem -t 'devices/#'
+```
+
## Reserved Topics
AWS IoT reserved topics such as `$aws/things/{thingName}/shadow/update` are service control topics, not ordinary application topics. Floci should handle these publishes by invoking IoT shadow behavior and then publishing the AWS-compatible response topics through the broker.
@@ -106,7 +118,7 @@ Implementation notes:
Current accepted limitation:
-- Certificate and policy authorization are not enforced at the broker layer yet.
+- Certificate and policy authorization are not enforced at the broker layer yet, on either port.
- Persistent offline sessions are not modeled yet.
- QoS 2 and advanced MQTT 5 property semantics remain follow-up scope.
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 0c1515f390..57155332a3 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -793,6 +793,13 @@ public interface EmulatorConfig {
@WithDefault("1883")
int port();
+
+ /**
+ * MQTT over TLS listener, the port AWS IoT serves for X.509 device connections. Opened only
+ * while {@code floci.tls.enabled} is true; {@code 0} disables it. Env: FLOCI_SERVICES_IOT_MQTT_TLS_PORT
+ */
+ @WithDefault("8883")
+ int tlsPort();
}
interface IotDataServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
index c5aab03718..3ee4b8bde4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
@@ -5,9 +5,16 @@ import io.github.hectorvent.floci.services.iot.model.IotRetainedMessage;
import io.netty.handler.codec.mqtt.MqttQoS;
import io.quarkus.runtime.ShutdownEvent;
import io.quarkus.runtime.StartupEvent;
+import io.quarkus.tls.CertificateUpdatedEvent;
+import io.quarkus.tls.TlsConfiguration;
+import io.quarkus.tls.TlsConfigurationRegistry;
+import io.quarkus.tls.runtime.config.TlsConfig;
import io.vertx.core.Vertx;
import io.vertx.core.buffer.Buffer;
+import io.vertx.core.http.ClientAuth;
+import io.vertx.core.net.KeyCertOptions;
import io.vertx.core.net.SocketAddress;
+import io.vertx.core.net.TrustOptions;
import io.vertx.mqtt.MqttEndpoint;
import io.vertx.mqtt.MqttServer;
import io.vertx.mqtt.MqttServerOptions;
@@ -20,6 +27,11 @@ import jakarta.enterprise.inject.Instance;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.X509ExtendedTrustManager;
+import java.net.Socket;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Base64;
import java.util.HashSet;
@@ -34,18 +46,63 @@ public class IotMqttBrokerService {
private static final Logger LOG = Logger.getLogger(IotMqttBrokerService.class);
+ /**
+ * The TLS listener asks for a client certificate and accepts whichever one is presented: AWS
+ * IoT trusts a device certificate because it is registered, not because of its issuer, and
+ * that lookup belongs to the broker, not to the handshake. Until it exists, 8883 admits every
+ * client exactly as 1883 does.
+ */
+ static final X509ExtendedTrustManager ACCEPT_ANY_CLIENT_CERTIFICATE = new X509ExtendedTrustManager() {
+ @Override
+ public void checkClientTrusted(X509Certificate[] chain, String authType) {
+ }
+
+ @Override
+ public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) {
+ }
+
+ @Override
+ public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine engine) {
+ }
+
+ @Override
+ public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+ throw new CertificateException("server-side trust manager");
+ }
+
+ @Override
+ public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException {
+ throw new CertificateException("server-side trust manager");
+ }
+
+ @Override
+ public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine engine) throws CertificateException {
+ throw new CertificateException("server-side trust manager");
+ }
+
+ @Override
+ public X509Certificate[] getAcceptedIssuers() {
+ return new X509Certificate[0];
+ }
+ };
+
private final EmulatorConfig config;
private final Vertx vertx;
private final Instance<IotService> iotService;
+ private final TlsConfigurationRegistry tlsRegistry;
private final Map<String, ClientSession> sessionsByClient = new ConcurrentHashMap<>();
private final Map<String, Map<String, Subscription>> subscriptionsByClient = new ConcurrentHashMap<>();
private MqttServer server;
+ private MqttServer tlsServer;
+ private ReloadingKeyManager keyManager;
@Inject
- public IotMqttBrokerService(EmulatorConfig config, Vertx vertx, Instance<IotService> iotService) {
+ public IotMqttBrokerService(EmulatorConfig config, Vertx vertx, Instance<IotService> iotService,
+ TlsConfigurationRegistry tlsRegistry) {
this.config = config;
this.vertx = vertx;
this.iotService = iotService;
+ this.tlsRegistry = tlsRegistry;
}
void onStart(@Observes StartupEvent ignored) {
@@ -72,33 +129,100 @@ public class IotMqttBrokerService {
return;
}
- MqttServer mqttServer = MqttServer.create(vertx, new MqttServerOptions()
+ MqttServer mqttServer = listen(new MqttServerOptions()
.setHost(config.services().iot().mqtt().host())
- .setPort(config.services().iot().mqtt().port()));
- mqttServer.endpointHandler(this::handleEndpoint);
- mqttServer.exceptionHandler(error -> LOG.warnv("IoT MQTT broker error: {0}", error.getMessage()));
+ .setPort(config.services().iot().mqtt().port()), "IoT MQTT broker");
+ try {
+ startTlsListener();
+ } catch (RuntimeException e) {
+ mqttServer.close().toCompletionStage().toCompletableFuture().join();
+ throw e;
+ }
+ server = mqttServer;
+ }
+ /**
+ * The MQTT over TLS listener, AWS IoT's 8883, sharing every handler with the plaintext one.
+ * Its key material is the TLS registry's default configuration, the same certificate the HTTPS
+ * endpoint presents. Not opened when the port is 0 or TLS is off.
+ */
+ private void startTlsListener() {
+ int tlsPort = config.services().iot().mqtt().tlsPort();
+ if (tlsPort <= 0) {
+ return;
+ }
+ if (!config.tls().enabled()) {
+ LOG.infov("IoT MQTT TLS listener not started on port {0}: floci.tls.enabled is false", Integer.toString(tlsPort));
+ return;
+ }
+ TlsConfiguration tls = tlsRegistry.getDefault().orElse(null);
+ if (tls == null) {
+ LOG.warnv("IoT MQTT TLS listener not started on port {0}: no default TLS configuration is registered",
+ Integer.toString(tlsPort));
+ return;
+ }
+ ReloadingKeyManager manager = new ReloadingKeyManager(ReloadingKeyManager.keyManagerOf(vertx, tls.getKeyStoreOptions()));
+ tlsServer = listen(new MqttServerOptions()
+ .setHost(config.services().iot().mqtt().host())
+ .setPort(tlsPort)
+ .setSsl(true)
+ .setKeyCertOptions(KeyCertOptions.wrap(manager))
+ .setTrustOptions(TrustOptions.wrap(ACCEPT_ANY_CLIENT_CERTIFICATE))
+ .setClientAuth(ClientAuth.REQUEST), "IoT MQTT TLS broker");
+ keyManager = manager;
+ }
+
+ private MqttServer listen(MqttServerOptions options, String name) {
+ MqttServer mqttServer = MqttServer.create(vertx, options);
+ mqttServer.endpointHandler(this::handleEndpoint);
+ mqttServer.exceptionHandler(error -> LOG.warnv("{0} error: {1}", name, error.getMessage()));
try {
mqttServer.listen().toCompletionStage().toCompletableFuture().join();
- server = mqttServer;
- LOG.infov("IoT MQTT broker started on {0}:{1}",
- config.services().iot().mqtt().host(), config.services().iot().mqtt().port());
} catch (Exception e) {
- mqttServer.close();
- throw new IllegalStateException("Failed to start IoT MQTT broker", e);
+ mqttServer.close().toCompletionStage().toCompletableFuture().join();
+ throw new IllegalStateException("Failed to start " + name + " on port " + options.getPort(), e);
+ }
+ LOG.infov("{0} started on {1}:{2}", name, options.getHost(), Integer.toString(options.getPort()));
+ return mqttServer;
+ }
+
+ /**
+ * The HTTPS server switches to a reissued server certificate through this event (a hostname
+ * learned by {@code TlsCertificateManager.ensureHost}, or a reload). The TLS listener follows
+ * by handing its key manager the reloaded key store: the next handshake presents the new
+ * certificate, established sessions keep theirs, and a failed reload keeps the previous one.
+ */
+ synchronized void onCertificateUpdated(@Observes CertificateUpdatedEvent event) {
+ ReloadingKeyManager manager = keyManager;
+ if (manager == null || !TlsConfig.DEFAULT_NAME.equalsIgnoreCase(event.name())) {
+ return;
+ }
+ try {
+ manager.reload(ReloadingKeyManager.keyManagerOf(vertx, event.tlsConfiguration().getKeyStoreOptions()));
+ LOG.debug("IoT MQTT TLS broker serves the reloaded server certificate");
+ } catch (Exception e) {
+ LOG.warnv(e, "IoT MQTT TLS broker keeps its previous server certificate: {0}", e.getMessage());
}
}
synchronized void stop() {
MqttServer mqttServer = server;
- if (mqttServer == null) {
+ MqttServer mqttTlsServer = tlsServer;
+ if (mqttServer == null && mqttTlsServer == null) {
return;
}
server = null;
+ tlsServer = null;
+ keyManager = null;
sessionsByClient.values().forEach(session -> session.endpoint().close());
sessionsByClient.clear();
subscriptionsByClient.clear();
- mqttServer.close().toCompletionStage().toCompletableFuture().join();
+ if (mqttServer != null) {
+ mqttServer.close().toCompletionStage().toCompletableFuture().join();
+ }
+ if (mqttTlsServer != null) {
+ mqttTlsServer.close().toCompletionStage().toCompletableFuture().join();
+ }
LOG.info("IoT MQTT broker stopped");
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/ReloadingKeyManager.java b/src/main/java/io/github/hectorvent/floci/services/iot/ReloadingKeyManager.java
new file mode 100644
index 0000000000..26911a31cd
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/ReloadingKeyManager.java
@@ -0,0 +1,172 @@
+package io.github.hectorvent.floci.services.iot;
+
+import io.vertx.core.Vertx;
+import io.vertx.core.net.KeyCertOptions;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.X509ExtendedKeyManager;
+import javax.net.ssl.X509KeyManager;
+import java.net.Socket;
+import java.security.Principal;
+import java.security.PrivateKey;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.LongSupplier;
+
+/**
+ * Server key manager for the MQTT TLS listener that answers every handshake from the key material
+ * it was last given, so a reissued server certificate is served without rebuilding the listener's
+ * SSL context: Vert.x drops the connections it accepts while a context is being rebuilt, whereas
+ * JSSE asks the key manager for an alias, then that alias's key and chain, on every handshake.
+ *
+ * <p>The alias handed to JSSE names the generation it came from, and a retired generation stays
+ * readable for {@link #RETIRED_GENERATION_RETENTION} after its replacement, longer than any
+ * handshake can live (Vert.x closes a connection whose handshake exceeds its 10 second timeout), so
+ * a handshake that began before any number of reloads still reads a matching key and chain.
+ * Retired generations are dropped on the next reload after that window.
+ */
+final class ReloadingKeyManager extends X509ExtendedKeyManager {
+
+ static final Duration RETIRED_GENERATION_RETENTION = Duration.ofSeconds(60);
+
+ private record Generation(long number, X509KeyManager delegate, long retiredAtNanos) {
+ }
+
+ private final LongSupplier nanoTime;
+ private volatile Generation current;
+ /** Superseded generations, newest first, each within the retention window at its last reload. */
+ private volatile List<Generation> retired = List.of();
+
+ ReloadingKeyManager(X509KeyManager delegate) {
+ this(delegate, System::nanoTime);
+ }
+
+ ReloadingKeyManager(X509KeyManager delegate, LongSupplier nanoTime) {
+ this.nanoTime = nanoTime;
+ this.current = new Generation(0, delegate, 0);
+ }
+
+ /** The first X.509 key manager of {@code options}, the TLS registry's default key store. */
+ static X509KeyManager keyManagerOf(Vertx vertx, KeyCertOptions options) {
+ if (options == null) {
+ throw new IllegalStateException("the default TLS configuration has no key store");
+ }
+ KeyManagerFactory factory;
+ try {
+ factory = options.getKeyManagerFactory(vertx);
+ } catch (Exception e) {
+ throw new IllegalStateException("the default TLS configuration's key store cannot be loaded: " + e.getMessage(), e);
+ }
+ if (factory != null) {
+ for (KeyManager manager : factory.getKeyManagers()) {
+ if (manager instanceof X509KeyManager x509) {
+ return x509;
+ }
+ }
+ }
+ throw new IllegalStateException("the default TLS configuration has no X.509 key manager");
+ }
+
+ /** Serves {@code delegate} from the next handshake on; the generations before it stay readable for a while. */
+ synchronized void reload(X509KeyManager delegate) {
+ long now = nanoTime.getAsLong();
+ List<Generation> kept = new ArrayList<>();
+ kept.add(new Generation(current.number(), current.delegate(), now));
+ for (Generation generation : retired) {
+ if (now - generation.retiredAtNanos() < RETIRED_GENERATION_RETENTION.toNanos()) {
+ kept.add(generation);
+ }
+ }
+ retired = List.copyOf(kept);
+ current = new Generation(current.number() + 1, delegate, 0);
+ }
+
+ int retiredGenerations() {
+ return retired.size();
+ }
+
+ @Override
+ public String chooseEngineServerAlias(String keyType, Principal[] issuers, SSLEngine engine) {
+ Generation generation = current;
+ return qualify(generation, generation.delegate() instanceof X509ExtendedKeyManager extended
+ ? extended.chooseEngineServerAlias(keyType, issuers, engine)
+ : generation.delegate().chooseServerAlias(keyType, issuers, null));
+ }
+
+ @Override
+ public String chooseServerAlias(String keyType, Principal[] issuers, Socket socket) {
+ Generation generation = current;
+ return qualify(generation, generation.delegate().chooseServerAlias(keyType, issuers, socket));
+ }
+
+ @Override
+ public String[] getServerAliases(String keyType, Principal[] issuers) {
+ Generation generation = current;
+ String[] aliases = generation.delegate().getServerAliases(keyType, issuers);
+ if (aliases == null) {
+ return new String[0];
+ }
+ String[] qualified = new String[aliases.length];
+ for (int i = 0; i < aliases.length; i++) {
+ qualified[i] = qualify(generation, aliases[i]);
+ }
+ return qualified;
+ }
+
+ @Override
+ public X509Certificate[] getCertificateChain(String alias) {
+ Generation generation = generationOf(alias);
+ return generation == null ? null : generation.delegate().getCertificateChain(unqualified(alias));
+ }
+
+ @Override
+ public PrivateKey getPrivateKey(String alias) {
+ Generation generation = generationOf(alias);
+ return generation == null ? null : generation.delegate().getPrivateKey(unqualified(alias));
+ }
+
+ @Override
+ public String[] getClientAliases(String keyType, Principal[] issuers) {
+ return new String[0];
+ }
+
+ @Override
+ public String chooseClientAlias(String[] keyType, Principal[] issuers, Socket socket) {
+ return null;
+ }
+
+ @Override
+ public String chooseEngineClientAlias(String[] keyType, Principal[] issuers, SSLEngine engine) {
+ return null;
+ }
+
+ private static String qualify(Generation generation, String alias) {
+ return alias == null ? null : generation.number() + ":" + alias;
+ }
+
+ private static String unqualified(String alias) {
+ return alias.substring(alias.indexOf(':') + 1);
+ }
+
+ private Generation generationOf(String alias) {
+ int separator = alias == null ? -1 : alias.indexOf(':');
+ if (separator <= 0) {
+ return null;
+ }
+ String number = alias.substring(0, separator);
+ Generation candidate = current;
+ if (Long.toString(candidate.number()).equals(number)) {
+ return candidate;
+ }
+ for (Generation generation : retired) {
+ if (Long.toString(generation.number()).equals(number)) {
+ return generation;
+ }
+ }
+ return null;
+ }
+}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 0cd7c50535..ba66846041 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -591,6 +591,7 @@ floci:
auto-start: false
host: 0.0.0.0
port: 1883
+ tls-port: 8883
iotdata:
enabled: true
cloudhsmv2:
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerServiceTest.java
new file mode 100644
index 0000000000..04898b22b0
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerServiceTest.java
@@ -0,0 +1,453 @@
+package io.github.hectorvent.floci.services.iot;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.config.FlociCertificateAuthority;
+import io.github.hectorvent.floci.services.acm.CertificateGenerator;
+import io.github.hectorvent.floci.services.acm.model.KeyAlgorithm;
+import io.quarkus.tls.CertificateUpdatedEvent;
+import io.quarkus.tls.TlsConfiguration;
+import io.quarkus.tls.TlsConfigurationRegistry;
+import io.quarkus.tls.runtime.config.TlsConfig;
+import io.vertx.core.Vertx;
+import io.vertx.core.buffer.Buffer;
+import io.vertx.core.net.KeyCertOptions;
+import io.vertx.core.net.PemKeyCertOptions;
+import jakarta.enterprise.inject.Instance;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.SSLSocket;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509ExtendedKeyManager;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.nio.file.Path;
+import java.security.KeyStore;
+import java.security.Principal;
+import java.security.PrivateKey;
+import java.security.cert.Certificate;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+/**
+ * The MQTT over TLS listener against a real Vert.x and a mocked TLS registry: what it serves,
+ * when it is not opened, how a bind failure is reported, and how a reloaded server certificate
+ * reaches the next handshake without a restart.
+ */
+class IotMqttBrokerServiceTest {
+
+ private static final CertificateGenerator GENERATOR = new CertificateGenerator();
+
+ /**
+ * AWS IoT's default security policy, IoTSecurityPolicy_TLS13_1_2_2022_10, in JSSE names: the
+ * TLS 1.3 suites and the RSA-authenticated TLS 1.2 suites it lists (transport-security page).
+ */
+ private static final Set<String> AWS_TLS13_SUITES = Set.of(
+ "TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384", "TLS_CHACHA20_POLY1305_SHA256");
+ private static final Set<String> AWS_TLS12_RSA_SUITES = Set.of(
+ "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256",
+ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+ "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384", "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",
+ "TLS_RSA_WITH_AES_128_GCM_SHA256", "TLS_RSA_WITH_AES_128_CBC_SHA256", "TLS_RSA_WITH_AES_128_CBC_SHA",
+ "TLS_RSA_WITH_AES_256_GCM_SHA384", "TLS_RSA_WITH_AES_256_CBC_SHA256", "TLS_RSA_WITH_AES_256_CBC_SHA");
+
+ private static Vertx vertx;
+ private static FlociCertificateAuthority ca;
+ private static CertificateGenerator.GeneratedCertificate bootLeaf;
+ private static CertificateGenerator.GeneratedCertificate reissuedLeaf;
+
+ private final EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ private final TlsConfigurationRegistry registry = mock(TlsConfigurationRegistry.class);
+ private final TlsConfiguration tls = mock(TlsConfiguration.class);
+ @SuppressWarnings("unchecked")
+ private final Instance<IotService> iotService = mock(Instance.class);
+ private int plainPort;
+ private int tlsPort;
+ private IotMqttBrokerService broker;
+
+ @BeforeAll
+ static void keyMaterial(@TempDir Path dir) {
+ vertx = Vertx.vertx();
+ ca = FlociCertificateAuthority.loadOrCreate(dir);
+ bootLeaf = ca.issueServerCertificate("localhost", List.of("localhost", "127.0.0.1"), KeyAlgorithm.RSA_2048, null);
+ reissuedLeaf = ca.issueServerCertificate("localhost",
+ List.of("localhost", "127.0.0.1", "iot.dev.localhost.floci.io"), KeyAlgorithm.RSA_2048, null);
+ }
+
+ @AfterAll
+ static void closeVertx() {
+ vertx.close().toCompletionStage().toCompletableFuture().join();
+ }
+
+ @BeforeEach
+ void brokerWithTlsOn() throws IOException {
+ plainPort = freePort();
+ do {
+ tlsPort = freePort();
+ } while (tlsPort == plainPort);
+ when(config.services().iot().enabled()).thenReturn(true);
+ when(config.services().iot().mqtt().enabled()).thenReturn(true);
+ when(config.services().iot().mqtt().host()).thenReturn("127.0.0.1");
+ when(config.services().iot().mqtt().port()).thenReturn(plainPort);
+ when(config.services().iot().mqtt().tlsPort()).thenReturn(tlsPort);
+ when(config.tls().enabled()).thenReturn(true);
+ when(registry.getDefault()).thenReturn(Optional.of(tls));
+ when(tls.getKeyStoreOptions()).thenReturn(pem(bootLeaf));
+ broker = new IotMqttBrokerService(config, vertx, iotService, registry);
+ }
+
+ @AfterEach
+ void stopBroker() {
+ broker.stop();
+ }
+
+ @Test
+ void tlsListenerServesTheRegistryLeafAndAsksForAClientCertificateWithoutRequiringOne() throws Exception {
+ broker.startIfEnabled();
+
+ RecordingKeyManager clientKeys = new RecordingKeyManager();
+ X509Certificate served = handshake(new KeyManager[] {clientKeys});
+
+ assertEquals(serial(bootLeaf), served.getSerialNumber(), "the leaf from the TLS registry's default configuration");
+ assertTrue(clientKeys.asked.get(), "the server sent a CertificateRequest");
+ assertTrue(accepts(plainPort), "the plaintext listener is up as well");
+ }
+
+ @Test
+ void aDeviceOnAwsDefaultSecurityPolicyNegotiatesTls13() throws Exception {
+ broker.startIfEnabled();
+
+ SSLSession session = negotiate("TLSv1.3", AWS_TLS13_SUITES);
+
+ assertEquals("TLSv1.3", session.getProtocol());
+ assertTrue(AWS_TLS13_SUITES.contains(session.getCipherSuite()), session.getCipherSuite());
+ }
+
+ @Test
+ void aDeviceOnAwsDefaultSecurityPolicyNegotiatesTls12() throws Exception {
+ broker.startIfEnabled();
+
+ SSLSession session = negotiate("TLSv1.2", AWS_TLS12_RSA_SUITES);
+
+ assertEquals("TLSv1.2", session.getProtocol());
+ assertTrue(AWS_TLS12_RSA_SUITES.contains(session.getCipherSuite()), session.getCipherSuite());
+ }
+
+ @Test
+ void aClientCertificateFromAnyIssuerCompletesTheHandshake() throws Exception {
+ broker.startIfEnabled();
+ var device = GENERATOR.generateSelfSignedCertificate("device", List.of(), KeyAlgorithm.RSA_2048);
+
+ X509Certificate served = handshake(keyManagers(device));
+
+ assertEquals(serial(bootLeaf), served.getSerialNumber());
+ }
+
+ @Test
+ void tlsPortZeroOpensOnlyThePlaintextListener() {
+ when(config.services().iot().mqtt().tlsPort()).thenReturn(0);
+
+ broker.startIfEnabled();
+
+ assertTrue(accepts(plainPort));
+ verifyNoInteractions(registry);
+ }
+
+ @Test
+ void tlsDisabledOpensOnlyThePlaintextListener() {
+ when(config.tls().enabled()).thenReturn(false);
+
+ broker.startIfEnabled();
+
+ assertTrue(accepts(plainPort));
+ assertFalse(accepts(tlsPort), "no TLS listener while floci.tls.enabled is false");
+ verifyNoInteractions(registry);
+ }
+
+ @Test
+ void noDefaultTlsConfigurationLeavesTheTlsPortClosed() {
+ when(registry.getDefault()).thenReturn(Optional.empty());
+
+ broker.startIfEnabled();
+
+ assertTrue(broker.isRunning());
+ assertTrue(accepts(plainPort));
+ assertFalse(accepts(tlsPort));
+ }
+
+ @Test
+ void aTlsBindFailureLeavesNoListenerBehindAndTheNextStartSucceeds() throws Exception {
+ try (ServerSocket blocker = new ServerSocket()) {
+ blocker.bind(new InetSocketAddress("127.0.0.1", tlsPort));
+
+ IllegalStateException failure = assertThrows(IllegalStateException.class, broker::startIfEnabled);
+
+ assertTrue(failure.getMessage().contains(Integer.toString(tlsPort)), failure.getMessage());
+ assertFalse(broker.isRunning());
+ awaitClosed(plainPort);
+ }
+
+ broker.startIfEnabled();
+
+ assertTrue(broker.isRunning());
+ assertTrue(accepts(plainPort));
+ assertEquals(serial(bootLeaf), handshake(null).getSerialNumber());
+ }
+
+ @Test
+ void certificateUpdatedEventSwapsTheLeafForTheNextHandshake() throws Exception {
+ broker.startIfEnabled();
+ assertEquals(serial(bootLeaf), handshake(null).getSerialNumber());
+ when(tls.getKeyStoreOptions()).thenReturn(pem(reissuedLeaf));
+
+ broker.onCertificateUpdated(new CertificateUpdatedEvent(TlsConfig.DEFAULT_NAME, tls));
+
+ assertEquals(serial(reissuedLeaf), handshake(null).getSerialNumber());
+ }
+
+ @Test
+ void anEventForAnotherTlsConfigurationIsIgnored() throws Exception {
+ broker.startIfEnabled();
+ when(tls.getKeyStoreOptions()).thenReturn(pem(reissuedLeaf));
+
+ broker.onCertificateUpdated(new CertificateUpdatedEvent("rds-proxy", tls));
+
+ assertEquals(serial(bootLeaf), handshake(null).getSerialNumber());
+ }
+
+ @Test
+ void aFailedSwapKeepsThePreviousLeaf() throws Exception {
+ broker.startIfEnabled();
+ when(tls.getKeyStoreOptions()).thenReturn(new PemKeyCertOptions()
+ .addCertValue(Buffer.buffer("not a certificate"))
+ .addKeyValue(Buffer.buffer("not a key")));
+
+ broker.onCertificateUpdated(new CertificateUpdatedEvent(TlsConfig.DEFAULT_NAME, tls));
+
+ assertEquals(serial(bootLeaf), handshake(null).getSerialNumber());
+ }
+
+ @Test
+ void eventsBeforeStartAndAfterStopAreIgnored() {
+ broker.onCertificateUpdated(new CertificateUpdatedEvent(TlsConfig.DEFAULT_NAME, tls));
+
+ broker.startIfEnabled();
+ broker.stop();
+ broker.onCertificateUpdated(new CertificateUpdatedEvent(TlsConfig.DEFAULT_NAME, tls));
+
+ assertFalse(broker.isRunning());
+ awaitClosed(tlsPort);
+ awaitClosed(plainPort);
+ }
+
+ @Test
+ void stopClosesBothListenersAndStartReopensThem() throws Exception {
+ broker.startIfEnabled();
+ broker.stop();
+ awaitClosed(tlsPort);
+ awaitClosed(plainPort);
+ when(tls.getKeyStoreOptions()).thenReturn(pem(reissuedLeaf));
+
+ broker.startIfEnabled();
+
+ assertTrue(accepts(plainPort));
+ assertEquals(serial(reissuedLeaf), handshake(null).getSerialNumber(), "a restart reads the registry again");
+ }
+
+ /**
+ * Reloads while connections arrive: every handshake completes with one of the two leaves.
+ * Rebuilding the listener's SSL context instead would drop the connections accepted while
+ * the new context is being built (Vert.x serves a pending update to them as null).
+ */
+ @Test
+ void concurrentCertificateEventsAndHandshakesAllSucceed() throws Exception {
+ broker.startIfEnabled();
+ when(tls.getKeyStoreOptions()).thenAnswer(ignored -> pem(reissuedLeaf));
+ int threads = 8;
+ int rounds = 10;
+ CountDownLatch start = new CountDownLatch(1);
+ ExecutorService pool = Executors.newFixedThreadPool(threads * 2);
+ List<Future<?>> outcomes = new ArrayList<>();
+ try {
+ for (int i = 0; i < threads; i++) {
+ outcomes.add(pool.submit(() -> {
+ start.await();
+ for (int round = 0; round < rounds; round++) {
+ broker.onCertificateUpdated(new CertificateUpdatedEvent(TlsConfig.DEFAULT_NAME, tls));
+ }
+ return null;
+ }));
+ outcomes.add(pool.submit(() -> {
+ start.await();
+ for (int round = 0; round < rounds; round++) {
+ X509Certificate served = handshake(null);
+ assertTrue(served.getSerialNumber().equals(serial(bootLeaf))
+ || served.getSerialNumber().equals(serial(reissuedLeaf)), "one of the two leaves");
+ }
+ return null;
+ }));
+ }
+ start.countDown();
+ for (Future<?> outcome : outcomes) {
+ outcome.get(30, TimeUnit.SECONDS);
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+
+ assertEquals(serial(reissuedLeaf), handshake(null).getSerialNumber());
+ }
+
+ private X509Certificate handshake(KeyManager[] clientKeys) throws Exception {
+ try (SSLSocket socket = connectTls(clientKeys)) {
+ socket.startHandshake();
+ return (X509Certificate) socket.getSession().getPeerCertificates()[0];
+ }
+ }
+
+ /** A handshake as a device pinned to one protocol version and AWS's suites for it would do. */
+ private SSLSession negotiate(String protocol, Set<String> suites) throws Exception {
+ try (SSLSocket socket = connectTls(null)) {
+ socket.setEnabledProtocols(new String[] {protocol});
+ socket.setEnabledCipherSuites(List.of(socket.getSupportedCipherSuites()).stream()
+ .filter(suites::contains).toArray(String[]::new));
+ socket.startHandshake();
+ return socket.getSession();
+ }
+ }
+
+ private SSLSocket connectTls(KeyManager[] clientKeys) throws Exception {
+ KeyStore trust = KeyStore.getInstance(KeyStore.getDefaultType());
+ trust.load(null, null);
+ trust.setCertificateEntry("floci", ca.certificate());
+ TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ tmf.init(trust);
+ SSLContext context = SSLContext.getInstance("TLS");
+ context.init(clientKeys, tmf.getTrustManagers(), null);
+ SSLSocket socket = (SSLSocket) context.getSocketFactory().createSocket();
+ socket.connect(new InetSocketAddress("127.0.0.1", tlsPort), 5_000);
+ return socket;
+ }
+
+ private static KeyManager[] keyManagers(CertificateGenerator.GeneratedCertificate leaf) throws Exception {
+ KeyStore keys = KeyStore.getInstance(KeyStore.getDefaultType());
+ keys.load(null, null);
+ PrivateKey key = GENERATOR.parsePrivateKey(leaf.privateKeyPem());
+ keys.setKeyEntry("device", key, new char[0], new Certificate[] {GENERATOR.parseCertificate(leaf.certificatePem())});
+ KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+ kmf.init(keys, new char[0]);
+ return kmf.getKeyManagers();
+ }
+
+ private static KeyCertOptions pem(CertificateGenerator.GeneratedCertificate leaf) {
+ return new PemKeyCertOptions()
+ .addCertValue(Buffer.buffer(leaf.certificatePem()))
+ .addKeyValue(Buffer.buffer(leaf.privateKeyPem()));
+ }
+
+ private static java.math.BigInteger serial(CertificateGenerator.GeneratedCertificate leaf) {
+ return GENERATOR.parseCertificate(leaf.certificatePem()).getSerialNumber();
+ }
+
+ private static boolean accepts(int port) {
+ try (Socket socket = new Socket()) {
+ socket.connect(new InetSocketAddress("127.0.0.1", port), 500);
+ return true;
+ } catch (IOException refused) {
+ return false;
+ }
+ }
+
+ /** Vert.x resolves close() a moment before the OS releases the port; poll like the lazy-start test. */
+ private static void awaitClosed(int port) {
+ long deadline = System.currentTimeMillis() + 5_000;
+ while (accepts(port)) {
+ if (System.currentTimeMillis() > deadline) {
+ throw new AssertionError("port " + port + " still accepts connections");
+ }
+ try {
+ Thread.sleep(50);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError("interrupted while waiting for port " + port + " to close", e);
+ }
+ }
+ }
+
+ private static int freePort() throws IOException {
+ try (ServerSocket socket = new ServerSocket(0)) {
+ return socket.getLocalPort();
+ }
+ }
+
+ /** A client key manager with no certificate that records whether the server asked for one. */
+ private static final class RecordingKeyManager extends X509ExtendedKeyManager {
+ final AtomicBoolean asked = new AtomicBoolean();
+
+ @Override
+ public String chooseClientAlias(String[] keyType, Principal[] issuers, Socket socket) {
+ asked.set(true);
+ return null;
+ }
+
+ @Override
+ public String chooseEngineClientAlias(String[] keyType, Principal[] issuers, SSLEngine engine) {
+ asked.set(true);
+ return null;
+ }
+
+ @Override
+ public String[] getClientAliases(String keyType, Principal[] issuers) {
+ return new String[0];
+ }
+
+ @Override
+ public String[] getServerAliases(String keyType, Principal[] issuers) {
+ return new String[0];
+ }
+
+ @Override
+ public String chooseServerAlias(String keyType, Principal[] issuers, Socket socket) {
+ return null;
+ }
+
+ @Override
+ public X509Certificate[] getCertificateChain(String alias) {
+ return null;
+ }
+
+ @Override
+ public PrivateKey getPrivateKey(String alias) {
+ return null;
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttTlsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttTlsIntegrationTest.java
new file mode 100644
index 0000000000..86d132079d
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttTlsIntegrationTest.java
@@ -0,0 +1,371 @@
+package io.github.hectorvent.floci.services.iot;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.CertificateMetadata;
+import io.github.hectorvent.floci.config.FlociCertificateAuthority;
+import io.github.hectorvent.floci.config.TlsCertificateManager;
+import io.github.hectorvent.floci.services.acm.CertificateGenerator;
+import io.github.hectorvent.floci.services.acm.model.KeyAlgorithm;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
+import org.eclipse.paho.client.mqttv3.MqttCallback;
+import org.eclipse.paho.client.mqttv3.MqttClient;
+import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
+import org.eclipse.paho.client.mqttv3.MqttException;
+import org.eclipse.paho.client.mqttv3.MqttMessage;
+import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
+import org.junit.jupiter.api.Test;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.SSLParameters;
+import javax.net.ssl.SSLSocket;
+import javax.net.ssl.TrustManagerFactory;
+import java.io.IOException;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.KeyStore;
+import java.security.cert.Certificate;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Devices connect to the MQTT over TLS listener the way they connect to AWS IoT's 8883: over
+ * {@code ssl://}, trusting one CA, verifying the endpoint name, with MQTT 3.1.1 or MQTT 5, and
+ * with the same broker behaviour as the plaintext listener.
+ */
+@QuarkusTest
+@TestProfile(IotMqttTlsIntegrationTest.Profile.class)
+class IotMqttTlsIntegrationTest {
+
+ static final Path DATA_DIR = Path.of("target", "floci-iot-mqtt-tls-test").toAbsolutePath();
+ static final Path TLS_DIR = DATA_DIR.resolve("tls");
+ static final int PLAIN_PORT = 18834;
+ static final int TLS_PORT = 18835;
+ static final String LEARNED_HOST = "iot.dev.localhost.floci.io";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ @Inject
+ TlsCertificateManager certificateManager;
+
+ @Inject
+ IotMqttBrokerService broker;
+
+ @Test
+ void connectSubscribePublishOverTlsTrustingOnlyTheFlociCa() throws Exception {
+ String topic = "tls/roundtrip/" + System.nanoTime();
+ byte[] payload = "hello over 8883".getBytes(StandardCharsets.UTF_8);
+
+ try (TlsClient subscriber = TlsClient.connect("tls-sub-" + System.nanoTime(), null)) {
+ subscriber.subscribe(topic, 1);
+ try (TlsClient publisher = TlsClient.connect("tls-pub-" + System.nanoTime(), null)) {
+ publisher.publish(topic, payload, 1);
+ }
+ assertArrayEquals(payload, subscriber.takePayload());
+ }
+ }
+
+ @Test
+ void mqtt5ConnectsOverTls() throws Exception {
+ org.eclipse.paho.mqttv5.client.MqttClient client = new org.eclipse.paho.mqttv5.client.MqttClient(
+ "ssl://127.0.0.1:" + TLS_PORT, "tls-v5-" + System.nanoTime(), null);
+ org.eclipse.paho.mqttv5.client.MqttConnectionOptions options =
+ new org.eclipse.paho.mqttv5.client.MqttConnectionOptions();
+ options.setCleanStart(true);
+ options.setConnectionTimeout(10);
+ options.setSocketFactory(trustOnlyTheCa(null).getSocketFactory());
+ try {
+ client.connect(options);
+ assertTrue(client.isConnected());
+ client.disconnect();
+ } finally {
+ client.close();
+ }
+ }
+
+ @Test
+ void aDeviceCertificateIssuedByTheCaIsAcceptedAndTheSessionIsVisibleToTheConnectionApi() throws Exception {
+ String clientId = "tls-device-" + System.nanoTime();
+ var device = FlociCertificateAuthority.loadOrCreate(TLS_DIR).issueClientCertificate("device-" + clientId);
+
+ try (TlsClient client = TlsClient.connect(clientId, keyManagers(device))) {
+ assertTrue(client.isConnected());
+ given()
+ .queryParam("includeSocketInformation", true)
+ .when()
+ .get("/connections/{clientId}", clientId)
+ .then()
+ .statusCode(200)
+ .body("connected", equalTo(true))
+ .body("sourceIp", equalTo("127.0.0.1"));
+ }
+ }
+
+ @Test
+ void plaintextOnTheTlsPortIsRefused() throws Exception {
+ MqttClient plain = new MqttClient("tcp://127.0.0.1:" + TLS_PORT, "plain-on-tls-" + System.nanoTime(), new MemoryPersistence());
+ try {
+ MqttConnectOptions options = new MqttConnectOptions();
+ options.setConnectionTimeout(5);
+
+ assertThrows(MqttException.class, () -> plain.connect(options));
+
+ assertFalse(plain.isConnected());
+ } finally {
+ plain.close();
+ }
+ }
+
+ @Test
+ void plaintextListenerStillWorks() throws Exception {
+ MqttClient client = new MqttClient("tcp://127.0.0.1:" + PLAIN_PORT, "plain-" + System.nanoTime(), new MemoryPersistence());
+ try {
+ client.connect();
+ assertTrue(client.isConnected());
+ client.disconnect();
+ } finally {
+ client.close();
+ }
+ }
+
+ @Test
+ void learnedHostnameIsPresentedOnTheNextTlsHandshake() throws Exception {
+ assertThrows(SSLHandshakeException.class, () -> handshakeVerifying(LEARNED_HOST),
+ "precondition: a device verifying the custom domain rejects the boot leaf");
+
+ certificateManager.ensureHost(LEARNED_HOST);
+
+ Set<String> sans = sans(handshakeVerifying(LEARNED_HOST));
+ assertTrue(sans.contains(LEARNED_HOST), sans.toString());
+ assertTrue(sans.contains("localhost"), "configured names are still served: " + sans);
+ }
+
+ @Test
+ void aClientIdReconnectingOnThePlaintextListenerTakesOverTheTlsSession() throws Exception {
+ String clientId = "tls-takeover-" + System.nanoTime();
+
+ try (TlsClient first = TlsClient.connect(clientId, null)) {
+ MqttClient second = new MqttClient("tcp://127.0.0.1:" + PLAIN_PORT, clientId, new MemoryPersistence());
+ try {
+ second.connect();
+ first.awaitDisconnected();
+ assertTrue(second.isConnected());
+ second.disconnect();
+ } finally {
+ second.close();
+ }
+ }
+ }
+
+ @Test
+ void shadowUpdateOverTlsPublishesTheAcceptedResponseOverTls() throws Exception {
+ String thing = "tlsThing" + System.nanoTime();
+ try (TlsClient subscriber = TlsClient.connect("tls-shadow-sub-" + System.nanoTime(), null)) {
+ subscriber.subscribe("$aws/things/" + thing + "/shadow/update/accepted", 0);
+ try (TlsClient publisher = TlsClient.connect("tls-shadow-pub-" + System.nanoTime(), null)) {
+ publisher.publish("$aws/things/" + thing + "/shadow/update",
+ "{\"state\":{\"desired\":{\"color\":\"blue\"}},\"clientToken\":\"tls-token\"}".getBytes(StandardCharsets.UTF_8), 0);
+ }
+ String accepted = new String(subscriber.takePayload(), StandardCharsets.UTF_8);
+ assertEquals("blue", OBJECT_MAPPER.readTree(accepted).path("state").path("desired").path("color").asText());
+ assertEquals("tls-token", OBJECT_MAPPER.readTree(accepted).path("clientToken").asText());
+ }
+ }
+
+ @Test
+ void aRestartOfTheBrokerServesTlsAgain() throws Exception {
+ broker.stop();
+ broker.startIfEnabled();
+
+ try (TlsClient client = TlsClient.connect("tls-restart-" + System.nanoTime(), null)) {
+ assertTrue(client.isConnected());
+ }
+ }
+
+ /**
+ * Connects by IP and hands JSSE the custom domain as the peer host, which is what a device SDK
+ * does once DNS points the name at Floci: SNI carries the name and the certificate must verify
+ * for it. The boot leaf's wildcard covers one label, so the three-label name is uncovered.
+ */
+ private static X509Certificate handshakeVerifying(String host) throws Exception {
+ try (Socket raw = new Socket("127.0.0.1", TLS_PORT);
+ SSLSocket socket = (SSLSocket) trustOnlyTheCa(null).getSocketFactory().createSocket(raw, host, TLS_PORT, true)) {
+ SSLParameters params = socket.getSSLParameters();
+ params.setEndpointIdentificationAlgorithm("HTTPS");
+ socket.setSSLParameters(params);
+ socket.startHandshake();
+ return (X509Certificate) socket.getSession().getPeerCertificates()[0];
+ }
+ }
+
+ private static SSLContext trustOnlyTheCa(KeyManager[] clientKeys) throws Exception {
+ KeyStore trust = KeyStore.getInstance(KeyStore.getDefaultType());
+ trust.load(null, null);
+ trust.setCertificateEntry("floci", FlociCertificateAuthority.loadOrCreate(TLS_DIR).certificate());
+ TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ tmf.init(trust);
+ SSLContext context = SSLContext.getInstance("TLS");
+ context.init(clientKeys, tmf.getTrustManagers(), null);
+ return context;
+ }
+
+ private static KeyManager[] keyManagers(CertificateGenerator.GeneratedCertificate leaf) throws Exception {
+ CertificateGenerator generator = new CertificateGenerator();
+ KeyStore keys = KeyStore.getInstance(KeyStore.getDefaultType());
+ keys.load(null, null);
+ keys.setKeyEntry("device", generator.parsePrivateKey(leaf.privateKeyPem()), new char[0],
+ new Certificate[] {generator.parseCertificate(leaf.certificatePem())});
+ KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+ kmf.init(keys, new char[0]);
+ return kmf.getKeyManagers();
+ }
+
+ private static Set<String> sans(X509Certificate leaf) throws Exception {
+ Set<String> sans = new TreeSet<>();
+ for (List<?> entry : leaf.getSubjectAlternativeNames()) {
+ sans.add(String.valueOf(entry.get(1)));
+ }
+ return sans;
+ }
+
+ /** A Paho MQTT 3.1.1 client on {@code ssl://}, with Paho's default hostname verification on. */
+ private static final class TlsClient implements AutoCloseable {
+ private final MqttClient client;
+ private final BlockingQueue<byte[]> payloads = new LinkedBlockingQueue<>();
+
+ private TlsClient(MqttClient client) {
+ this.client = client;
+ }
+
+ static TlsClient connect(String clientId, KeyManager[] clientKeys) throws Exception {
+ MqttClient client = new MqttClient("ssl://127.0.0.1:" + TLS_PORT, clientId, new MemoryPersistence());
+ TlsClient tlsClient = new TlsClient(client);
+ client.setCallback(new MqttCallback() {
+ @Override
+ public void connectionLost(Throwable cause) {
+ }
+
+ @Override
+ public void messageArrived(String topic, MqttMessage message) {
+ tlsClient.payloads.add(message.getPayload());
+ }
+
+ @Override
+ public void deliveryComplete(IMqttDeliveryToken token) {
+ }
+ });
+ MqttConnectOptions options = new MqttConnectOptions();
+ options.setCleanSession(true);
+ options.setConnectionTimeout(10);
+ options.setAutomaticReconnect(false);
+ options.setSocketFactory(trustOnlyTheCa(clientKeys).getSocketFactory());
+ client.connect(options);
+ return tlsClient;
+ }
+
+ boolean isConnected() {
+ return client.isConnected();
+ }
+
+ void subscribe(String topic, int qos) throws MqttException {
+ client.subscribe(topic, qos);
+ }
+
+ void publish(String topic, byte[] payload, int qos) throws MqttException {
+ client.publish(topic, payload, qos, false);
+ }
+
+ byte[] takePayload() throws InterruptedException {
+ byte[] payload = payloads.poll(10, TimeUnit.SECONDS);
+ assertNotNull(payload, "no publish received over TLS within 10s");
+ return payload;
+ }
+
+ void awaitDisconnected() throws InterruptedException {
+ Instant deadline = Instant.now().plus(Duration.ofSeconds(5));
+ while (Instant.now().isBefore(deadline)) {
+ if (!client.isConnected()) {
+ return;
+ }
+ Thread.sleep(25);
+ }
+ throw new AssertionError("the TLS session stayed open after its client id reconnected elsewhere");
+ }
+
+ @Override
+ public void close() throws MqttException {
+ if (client.isConnected()) {
+ client.disconnect();
+ }
+ client.close();
+ }
+ }
+
+ /**
+ * TLS on with a CA-issued leaf on disk, as TlsConfigSource leaves it in production. The
+ * bootstrap reads system properties, not profile overrides, so the profile lays the files
+ * down itself and points the TLS registry's default entry at them. The leaf carries
+ * {@code 127.0.0.1} so Paho's hostname verification passes against {@code ssl://127.0.0.1}.
+ */
+ public static final class Profile implements QuarkusTestProfile {
+
+ static {
+ try {
+ Files.createDirectories(TLS_DIR);
+ for (String name : List.of("floci-server.crt", "floci-server.key", "floci-server.metadata.json")) {
+ Files.deleteIfExists(TLS_DIR.resolve(name));
+ }
+ FlociCertificateAuthority ca = FlociCertificateAuthority.loadOrCreate(TLS_DIR);
+ List<String> sans = List.of("localhost", "127.0.0.1", "*.localhost.floci.io");
+ var leaf = ca.issueServerCertificate("localhost", sans, KeyAlgorithm.RSA_2048, null);
+ Files.writeString(TLS_DIR.resolve("floci-server.crt"), leaf.certificatePem());
+ Files.writeString(TLS_DIR.resolve("floci-server.key"), leaf.privateKeyPem());
+ Files.writeString(TLS_DIR.resolve("floci-server.metadata.json"),
+ OBJECT_MAPPER.writeValueAsString(CertificateMetadata.create(sans, "dev")));
+ } catch (IOException e) {
+ throw new IllegalStateException("could not prepare the TLS fixtures", e);
+ }
+ }
+
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return Map.ofEntries(
+ Map.entry("floci.tls.enabled", "true"),
+ Map.entry("floci.tls.self-signed", "true"),
+ Map.entry("floci.tls.aws-https-port", "0"),
+ Map.entry("floci.storage.persistent-path", DATA_DIR.toString()),
+ Map.entry("quarkus.tls.key-store.pem.0.cert", TLS_DIR.resolve("floci-server.crt").toString()),
+ Map.entry("quarkus.tls.key-store.pem.0.key", TLS_DIR.resolve("floci-server.key").toString()),
+ Map.entry("quarkus.http.insecure-requests", "enabled"),
+ Map.entry("floci.services.iot.mqtt.enabled", "true"),
+ Map.entry("floci.services.iot.mqtt.auto-start", "true"),
+ Map.entry("floci.services.iot.mqtt.host", "127.0.0.1"),
+ Map.entry("floci.services.iot.mqtt.port", Integer.toString(PLAIN_PORT)),
+ Map.entry("floci.services.iot.mqtt.tls-port", Integer.toString(TLS_PORT)));
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/ReloadingKeyManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/ReloadingKeyManagerTest.java
new file mode 100644
index 0000000000..5220ba19e8
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/ReloadingKeyManagerTest.java
@@ -0,0 +1,189 @@
+package io.github.hectorvent.floci.services.iot;
+
+import io.github.hectorvent.floci.services.acm.CertificateGenerator;
+import io.github.hectorvent.floci.services.acm.model.KeyAlgorithm;
+import io.vertx.core.Vertx;
+import io.vertx.core.buffer.Buffer;
+import io.vertx.core.net.PemKeyCertOptions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.X509ExtendedKeyManager;
+import java.security.PrivateKey;
+import java.security.cert.X509Certificate;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * JSSE asks a key manager for an alias, then for that alias's key and chain, on every handshake.
+ * The manager under test answers from its current delegate and keeps a handshake that started
+ * before a reload on the delegate it began with.
+ */
+class ReloadingKeyManagerTest {
+
+ private static Vertx vertx;
+
+ @BeforeAll
+ static void startVertx() {
+ vertx = Vertx.vertx();
+ }
+
+ @AfterAll
+ static void closeVertx() {
+ vertx.close().toCompletionStage().toCompletableFuture().join();
+ }
+
+ @Test
+ void serverAliasAndItsMaterialComeFromTheDelegate() {
+ X509ExtendedKeyManager inner = delegate("srv", "first");
+
+ ReloadingKeyManager manager = new ReloadingKeyManager(inner);
+
+ String alias = manager.chooseEngineServerAlias("RSA", null, null);
+ assertNotNull(alias);
+ assertSame(inner.getCertificateChain("srv"), manager.getCertificateChain(alias));
+ assertSame(inner.getPrivateKey("srv"), manager.getPrivateKey(alias));
+ assertEquals(alias, manager.chooseServerAlias("RSA", null, null), "socket and engine flavours agree");
+ assertArrayEquals(new String[] {alias}, manager.getServerAliases("RSA", null));
+ }
+
+ @Test
+ void reloadServesTheNewDelegateToTheNextHandshake() {
+ X509ExtendedKeyManager first = delegate("srv", "first");
+ X509ExtendedKeyManager second = delegate("srv", "second");
+ ReloadingKeyManager manager = new ReloadingKeyManager(first);
+
+ manager.reload(second);
+
+ String alias = manager.chooseEngineServerAlias("RSA", null, null);
+ assertSame(second.getCertificateChain("srv"), manager.getCertificateChain(alias));
+ assertSame(second.getPrivateKey("srv"), manager.getPrivateKey(alias));
+ }
+
+ @Test
+ void anAliasChosenBeforeAReloadKeepsItsOwnChainAndKey() {
+ X509ExtendedKeyManager first = delegate("srv", "first");
+ X509ExtendedKeyManager second = delegate("srv", "second");
+ ReloadingKeyManager manager = new ReloadingKeyManager(first);
+ String inFlight = manager.chooseEngineServerAlias("RSA", null, null);
+
+ manager.reload(second);
+
+ assertSame(first.getCertificateChain("srv"), manager.getCertificateChain(inFlight));
+ assertSame(first.getPrivateKey("srv"), manager.getPrivateKey(inFlight));
+ }
+
+ @Test
+ void anAliasChosenBeforeSeveralQuickReloadsStillResolvesItsOwnChainAndKey() {
+ X509ExtendedKeyManager first = delegate("srv", "first");
+ ReloadingKeyManager manager = new ReloadingKeyManager(first);
+ String inFlight = manager.chooseEngineServerAlias("RSA", null, null);
+
+ manager.reload(delegate("srv", "second"));
+ manager.reload(delegate("srv", "third"));
+ manager.reload(delegate("srv", "fourth"));
+
+ assertSame(first.getCertificateChain("srv"), manager.getCertificateChain(inFlight));
+ assertSame(first.getPrivateKey("srv"), manager.getPrivateKey(inFlight));
+ }
+
+ @Test
+ void aRetiredGenerationIsDroppedOnceNoHandshakeCanStillReferenceIt() {
+ AtomicLong clock = new AtomicLong();
+ X509ExtendedKeyManager first = delegate("srv", "first");
+ ReloadingKeyManager manager = new ReloadingKeyManager(first, clock::get);
+ String stale = manager.chooseEngineServerAlias("RSA", null, null);
+ manager.reload(delegate("srv", "second"));
+ clock.addAndGet(ReloadingKeyManager.RETIRED_GENERATION_RETENTION.toNanos() - 1);
+ manager.reload(delegate("srv", "third"));
+ assertSame(first.getPrivateKey("srv"), manager.getPrivateKey(stale), "still inside the retention window");
+
+ clock.addAndGet(1);
+ manager.reload(delegate("srv", "fourth"));
+
+ assertNull(manager.getCertificateChain(stale));
+ assertNull(manager.getPrivateKey(stale));
+ assertEquals(2, manager.retiredGenerations(), "the two generations retired inside the window are kept, the first is gone");
+ }
+
+ @Test
+ void unknownAliasesAndKeyTypesYieldNull() {
+ ReloadingKeyManager manager = new ReloadingKeyManager(delegate("srv", "first"));
+
+ assertNull(manager.chooseEngineServerAlias("EC", null, null));
+ assertNull(manager.chooseServerAlias("EC", null, null));
+ assertEquals(0, manager.getServerAliases("EC", null).length);
+ assertNull(manager.getCertificateChain("garbage"));
+ assertNull(manager.getPrivateKey("garbage"));
+ assertNull(manager.getCertificateChain("999:srv"));
+ assertNull(manager.getPrivateKey(null));
+ }
+
+ @Test
+ void neverActsAsATlsClient() {
+ ReloadingKeyManager manager = new ReloadingKeyManager(delegate("srv", "first"));
+
+ assertNull(manager.chooseClientAlias(new String[] {"RSA"}, null, null));
+ assertNull(manager.chooseEngineClientAlias(new String[] {"RSA"}, null, null));
+ assertEquals(0, manager.getClientAliases("RSA", null).length);
+ }
+
+ @Test
+ void keyManagerOfLoadsThePemKeyStoreOfTheOptions() {
+ var leaf = new CertificateGenerator().generateSelfSignedCertificate("localhost", List.of("localhost"), KeyAlgorithm.RSA_2048);
+ PemKeyCertOptions options = new PemKeyCertOptions()
+ .addCertValue(Buffer.buffer(leaf.certificatePem()))
+ .addKeyValue(Buffer.buffer(leaf.privateKeyPem()));
+
+ ReloadingKeyManager manager = new ReloadingKeyManager(ReloadingKeyManager.keyManagerOf(vertx, options));
+
+ String alias = manager.chooseEngineServerAlias("RSA", null, null);
+ assertNotNull(alias);
+ assertEquals(new CertificateGenerator().parseCertificate(leaf.certificatePem()), manager.getCertificateChain(alias)[0]);
+ assertNotNull(manager.getPrivateKey(alias));
+ }
+
+ @Test
+ void keyManagerOfRefusesAConfigurationWithoutAKeyStore() {
+ IllegalStateException refused = assertThrows(IllegalStateException.class,
+ () -> ReloadingKeyManager.keyManagerOf(vertx, null));
+
+ assertEquals("the default TLS configuration has no key store", refused.getMessage());
+ }
+
+ @Test
+ void keyManagerOfReportsAKeyStoreThatCannotBeLoaded() {
+ PemKeyCertOptions broken = new PemKeyCertOptions()
+ .addCertValue(Buffer.buffer("not a certificate"))
+ .addKeyValue(Buffer.buffer("not a key"));
+
+ IllegalStateException refused = assertThrows(IllegalStateException.class,
+ () -> ReloadingKeyManager.keyManagerOf(vertx, broken));
+
+ assertNotNull(refused.getCause());
+ }
+
+ /** A delegate with one server alias for RSA whose chain and key are distinct mock objects. */
+ private static X509ExtendedKeyManager delegate(String alias, String label) {
+ X509ExtendedKeyManager inner = mock(X509ExtendedKeyManager.class, label);
+ X509Certificate cert = mock(X509Certificate.class, label + "-cert");
+ PrivateKey key = mock(PrivateKey.class, label + "-key");
+ when(inner.chooseEngineServerAlias("RSA", null, null)).thenReturn(alias);
+ when(inner.chooseServerAlias("RSA", null, null)).thenReturn(alias);
+ when(inner.getServerAliases("RSA", null)).thenReturn(new String[] {alias});
+ when(inner.getCertificateChain(alias)).thenReturn(new X509Certificate[] {cert});
+ when(inner.getPrivateKey(alias)).thenReturn(key);
+ return inner;
+ }
+}
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index 4ff6210d39..d565899684 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -354,6 +354,7 @@ floci:
auto-start: false
host: 127.0.0.1
port: 1883
+ tls-port: 8883
iotdata:
enabled: true
cloudhsmv2: