Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(iot): publish connection lifecycle events (#5277)

Mario Toffia87aa6f38a1b2
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/IotTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/IotTest.java
index 24ba98d448..837e846e05 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/IotTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/IotTest.java
@@ -135,6 +135,7 @@ import software.amazon.awssdk.services.iotjobsdataplane.model.UpdateJobExecution
import software.amazon.awssdk.services.sqs.SqsClient;
import software.amazon.awssdk.services.sqs.model.CreateQueueRequest;
import software.amazon.awssdk.services.sqs.model.DeleteQueueRequest;
+import software.amazon.awssdk.services.sqs.model.Message;
import software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest;
import software.amazon.awssdk.services.sqs.model.ReceiveMessageResponse;
@@ -149,6 +150,7 @@ import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -1067,6 +1069,75 @@ class IotTest {
}
}
+ @Test
+ void mqttPresenceEventsReachTopicRulesOverSqs() throws Exception {
+ String suffix = Long.toString(System.nanoTime());
+ String clientId = "tklocal-java-" + suffix;
+ String queueUrl = sqs.createQueue(CreateQueueRequest.builder()
+ .queueName("java-iot-presence-" + suffix)
+ .build()).queueUrl();
+ String connectedRule = "java_iot_presence_connected_" + suffix;
+ String disconnectedRule = "java_iot_presence_disconnected_" + suffix;
+ createSqsRule(connectedRule,
+ "SELECT * FROM '$aws/events/presence/connected/+' WHERE startswith(clientId, 'tklocal-')", queueUrl);
+ createSqsRule(disconnectedRule,
+ "SELECT * FROM '$aws/events/presence/disconnected/+' WHERE startswith(clientId, 'tklocal-')", queueUrl);
+ try {
+ try (Socket client = mqttConnect(clientId)) {
+ client.getOutputStream().write(new byte[] {(byte) 0xe0, 0x00});
+ }
+
+ Map<String, JsonNode> byType = new HashMap<>();
+ long deadline = System.currentTimeMillis() + 10_000;
+ while (byType.size() < 2 && System.currentTimeMillis() < deadline) {
+ for (Message message : sqs.receiveMessage(ReceiveMessageRequest.builder()
+ .queueUrl(queueUrl)
+ .maxNumberOfMessages(10)
+ .build()).messages()) {
+ JsonNode event = OBJECT_MAPPER.readTree(message.body());
+ if (clientId.equals(event.path("clientId").asText())) {
+ byType.put(event.path("eventType").asText(), event);
+ }
+ }
+ Thread.sleep(100);
+ }
+
+ assertThat(byType).containsOnlyKeys("connected", "disconnected");
+ JsonNode connected = byType.get("connected");
+ JsonNode disconnected = byType.get("disconnected");
+ assertThat(connected.path("timestamp").isIntegralNumber()).isTrue();
+ assertThat(connected.path("ipAddress").asText()).isNotEmpty();
+ assertThat(disconnected.path("sessionIdentifier").asText())
+ .isNotEmpty()
+ .isEqualTo(connected.path("sessionIdentifier").asText());
+ assertThat(disconnected.path("versionNumber").asLong()).isEqualTo(connected.path("versionNumber").asLong());
+ assertThat(disconnected.path("clientInitiatedDisconnect").asBoolean()).isTrue();
+ assertThat(disconnected.path("disconnectReason").asText()).isEqualTo("CLIENT_INITIATED_DISCONNECT");
+ } finally {
+ iot.deleteTopicRule(DeleteTopicRuleRequest.builder().ruleName(connectedRule).build());
+ iot.deleteTopicRule(DeleteTopicRuleRequest.builder().ruleName(disconnectedRule).build());
+ sqs.deleteQueue(DeleteQueueRequest.builder().queueUrl(queueUrl).build());
+ }
+ }
+
+ private void createSqsRule(String ruleName, String sql, String queueUrl) {
+ iot.createTopicRule(CreateTopicRuleRequest.builder()
+ .ruleName(ruleName)
+ .topicRulePayload(TopicRulePayload.builder()
+ .sql(sql)
+ .awsIotSqlVersion("2016-03-23")
+ .ruleDisabled(false)
+ .actions(Action.builder()
+ .sqs(SqsAction.builder()
+ .roleArn(TestFixtures.globalArn("iam", "000000000000", "role/iot-rule-role"))
+ .queueUrl(queueUrl)
+ .useBase64(false)
+ .build())
+ .build())
+ .build())
+ .build());
+ }
+
private Socket mqttConnect(String clientId) throws IOException {
Socket socket = new Socket();
socket.connect(new InetSocketAddress("floci", 1883), 5_000);
diff --git a/docs/services/iot.md b/docs/services/iot.md
index 417133775c..99d409a846 100644
--- a/docs/services/iot.md
+++ b/docs/services/iot.md
@@ -248,6 +248,49 @@ username, the URL query or the `x-amz-customauthorizer-name` upgrade header, see
[Authorizers](#authorizers); it does not verify a SigV4 signature, and a CONNECT that names no
authorizer is accepted as is.
+### Lifecycle events
+
+The broker publishes AWS IoT's lifecycle events for every MQTT session, on 1883, 8883 and `/mqtt`:
+`$aws/events/presence/connected/<clientId>` once the client is accepted, and
+`$aws/events/presence/disconnected/<clientId>` once its session ends. The payloads carry AWS's fields
+in AWS's order:
+
+```json
+{"clientId":"sensor-1","timestamp":1791399692873,"eventType":"connected","sessionIdentifier":"64b3ed16-3f27-4a75-8dfc-388884c2c216","principalIdentifier":"<certificateId>","versionNumber":0,"ipAddress":"203.0.113.7"}
+{"clientId":"sensor-1","timestamp":1791399695210,"eventType":"disconnected","sessionIdentifier":"64b3ed16-3f27-4a75-8dfc-388884c2c216","principalIdentifier":"<certificateId>","versionNumber":0,"clientInitiatedDisconnect":true,"disconnectReason":"CLIENT_INITIATED_DISCONNECT"}
+```
+
+- `timestamp` is epoch milliseconds.
+- `sessionIdentifier` is a random UUID, the same on the connected and the disconnected event of one session.
+- `principalIdentifier` is the device certificate ID on 8883. It is omitted on 1883, on `/mqtt` (Floci does not verify SigV4) and for a connection admitted by a custom authorizer.
+- `ipAddress`, on the connected event only, is the client's address. A WebSocket client reports its own address, not the bridge's loopback socket; `GetConnection` returns the same address, with the WebSocket client's own port as `sourcePort`.
+- `versionNumber` counts per client id: `0` for a new client id, `+2` when the client id reconnects after its session ended (`0`, `2`, `4`), and `+1` for a session that takes over the client id from a connected one. AWS starts again from `0` after about an hour without a connection; Floci keeps counting, across state resets, until the process restarts.
+
+`clientInitiatedDisconnect` is `true` only for `CLIENT_INITIATED_DISCONNECT`:
+
+| Cause | `disconnectReason` |
+|---|---|
+| The client sent `DISCONNECT` (MQTT 3.1.1 or 5) | `CLIENT_INITIATED_DISCONNECT` |
+| The connection closed without `DISCONNECT` | `CONNECTION_LOST` |
+| No packet from the client for 1.5 times its keep-alive | `MQTT_KEEP_ALIVE_TIMEOUT` |
+| A new connection took over the client id; published for the old session, before the new session's connected event | `DUPLICATE_CLIENTID` |
+| IoT Data `DeleteConnection` | `API_INITIATED_DISCONNECT` |
+| A PUBLISH above 128 KB, or a QoS 2 PUBLISH | `CLIENT_ERROR` |
+
+An event goes the way a client publish goes: it is evaluated by the topic rules of every region,
+where `clientid()` returns the client id the event is about and `topic(3)` is `presence`, and
+delivered to matching MQTT subscribers. As on AWS, a topic filter whose first level is `#` or `+`
+does not match the lifecycle topics, both for subscriptions and for a rule's `FROM`: `#` and
+`+/events/presence/+/+` receive no lifecycle events, `$aws/events/presence/connected/+` does.
+
+As on AWS, events can arrive out of order across connections; `versionNumber` orders them.
+
+Differences from AWS and limits:
+
+- A client id containing `#` or `+` gets no lifecycle events, as on AWS.
+- A broker shutdown publishes no disconnected events.
+- `connect_failed`, `subscribed` and `unsubscribed` events are not emitted.
+
## Reserved Topics
AWS IoT reserved topics such as `$aws/things/{thingName}/shadow/update` are service control topics, not ordinary application topics. Floci should handle these publishes by invoking IoT shadow behavior and then publishing the AWS-compatible response topics through the broker.
@@ -285,7 +328,7 @@ Implementation notes:
- Normal client publishes call `IotService.publish(...)` so retained-message storage, event recording, and rule evaluation remain service-owned.
- An MQTT publish stores the retained message and records the publish on the event loop, then hands topic rule evaluation to a single-thread worker the broker owns, which runs the rules of MQTT publishes one at a time in the order they arrived. A slow rule action therefore delays the rules of later publishes from every connection, but not their PUBACK or fan-out, and not other services' work.
- A state reset or a broker shutdown skips the rules of MQTT publishes still waiting, and an MQTT publish received during the reset or while the broker is stopped never runs its rules. An evaluation already running finishes.
-- Internal broker publishes fan out only to MQTT subscribers and do not recursively evaluate IoT topic rules.
+- Shadow responses the broker publishes internally fan out only to MQTT subscribers and do not recursively evaluate IoT topic rules. [Lifecycle events](#lifecycle-events) do evaluate topic rules.
Current accepted limitation:
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotCustomAuthorizer.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotCustomAuthorizer.java
index a6c2fb21b3..a65f6cf1b4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotCustomAuthorizer.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotCustomAuthorizer.java
@@ -57,9 +57,11 @@ public class IotCustomAuthorizer {
/**
* What a WebSocket upgrade carried: the header names lower-cased, the raw query without its
* {@code ?} ({@code ""} when there was none), the server name, null unless it came over TLS, and
- * the client's IP address, as the broker sees only the bridge's loopback socket.
+ * the client's IP address and port ({@code -1} when unknown), as the broker sees only the
+ * bridge's loopback socket.
*/
- record WebSocketUpgrade(Map<String, String> headers, String queryString, String serverName, String sourceIp) {
+ record WebSocketUpgrade(Map<String, String> headers, String queryString, String serverName, String sourceIp,
+ int sourcePort) {
}
@Inject
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
index a49184a788..1bf0c3dad7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
@@ -1,8 +1,11 @@
package io.github.hectorvent.floci.services.iot;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.services.iot.model.IotRetainedMessage;
+import io.netty.handler.codec.TooLongFrameException;
import io.netty.handler.codec.mqtt.MqttConnectReturnCode;
import io.netty.handler.codec.mqtt.MqttProperties;
import io.netty.handler.codec.mqtt.MqttQoS;
@@ -40,6 +43,7 @@ import javax.net.ssl.SSLPeerUnverifiedException;
import javax.net.ssl.SSLSession;
import javax.net.ssl.X509ExtendedTrustManager;
import java.net.Socket;
+import java.nio.charset.StandardCharsets;
import java.security.cert.Certificate;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
@@ -52,9 +56,11 @@ import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.LongSupplier;
import java.util.regex.Pattern;
@ApplicationScoped
@@ -113,6 +119,15 @@ public class IotMqttBrokerService implements Resettable {
static final int MAX_PENDING_RULE_EVALUATIONS = 1_000;
+ static final String CLIENT_INITIATED_DISCONNECT = "CLIENT_INITIATED_DISCONNECT";
+ static final String CONNECTION_LOST = "CONNECTION_LOST";
+ static final String MQTT_KEEP_ALIVE_TIMEOUT = "MQTT_KEEP_ALIVE_TIMEOUT";
+ static final String PRESENCE_TOPIC_PREFIX = "$aws/events/presence/";
+ private static final String DUPLICATE_CLIENTID = "DUPLICATE_CLIENTID";
+ private static final String API_INITIATED_DISCONNECT = "API_INITIATED_DISCONNECT";
+ private static final String CLIENT_ERROR = "CLIENT_ERROR";
+ private static final long KEEP_ALIVE_SLACK_NANOS = TimeUnit.MILLISECONDS.toNanos(250);
+
private static final long RULES_RESETTING = 1;
private static final long RULES_STOPPED = 2;
private static final long RULE_STATE_STEP = 4;
@@ -122,9 +137,14 @@ public class IotMqttBrokerService implements Resettable {
private final Instance<IotService> iotService;
private final IotCustomAuthorizer customAuthorizer;
private final TlsConfigurationRegistry tlsRegistry;
+ private final LongSupplier nanoTime;
final WorkerExecutor ruleWorker;
private final Map<String, ClientSession> sessionsByClient = new ConcurrentHashMap<>();
private final Map<String, Map<String, Subscription>> subscriptionsByClient = new ConcurrentHashMap<>();
+ /** Serializes admitting and ending sessions and the versionNumbers they take; never held across endpoint I/O. */
+ private final Object sessionLock = new Object();
+ /** The last lifecycle event versionNumber per client id; neither a state reset nor a broker restart clears it. */
+ private final Map<String, Long> presenceVersions = new ConcurrentHashMap<>();
private final AtomicInteger pendingRuleEvaluations = new AtomicInteger();
private final AtomicBoolean ruleEvaluationsThrottled = new AtomicBoolean();
/**
@@ -141,11 +161,17 @@ public class IotMqttBrokerService implements Resettable {
@Inject
public IotMqttBrokerService(EmulatorConfig config, Vertx vertx, Instance<IotService> iotService,
IotCustomAuthorizer customAuthorizer, TlsConfigurationRegistry tlsRegistry) {
+ this(config, vertx, iotService, customAuthorizer, tlsRegistry, System::nanoTime);
+ }
+
+ IotMqttBrokerService(EmulatorConfig config, Vertx vertx, Instance<IotService> iotService,
+ IotCustomAuthorizer customAuthorizer, TlsConfigurationRegistry tlsRegistry, LongSupplier nanoTime) {
this.config = config;
this.vertx = vertx;
this.iotService = iotService;
this.customAuthorizer = customAuthorizer;
this.tlsRegistry = tlsRegistry;
+ this.nanoTime = nanoTime;
this.ruleWorker = vertx.createSharedWorkerExecutor("floci-iot-mqtt-rules", 1);
}
@@ -305,11 +331,9 @@ public class IotMqttBrokerService implements Resettable {
tlsServer = null;
keyManager = null;
sessionsByClient.values().forEach(session -> {
- try {
- session.endpoint().close();
- } catch (IllegalStateException e) {
- LOG.debugv("IoT MQTT client {0} was already closed during broker shutdown", session.clientId());
- }
+ // Ended before the close, so a broker shutdown publishes no disconnected event.
+ session.ended().set(true);
+ closeEndpoint(session);
});
sessionsByClient.clear();
subscriptionsByClient.clear();
@@ -348,17 +372,28 @@ public class IotMqttBrokerService implements Resettable {
}
boolean disconnectClient(String clientId, boolean cleanSession) {
- ClientSession session = sessionsByClient.remove(clientId);
- if (session == null) {
- return false;
- }
- if (cleanSession) {
- subscriptionsByClient.remove(clientId);
+ ClientSession session;
+ synchronized (sessionLock) {
+ session = sessionsByClient.get(clientId);
+ if (session == null || !endSession(session, API_INITIATED_DISCONNECT)) {
+ return false;
+ }
+ if (cleanSession) {
+ subscriptionsByClient.remove(clientId);
+ }
}
- session.endpoint().close();
+ closeEndpoint(session);
return true;
}
+ private static void closeEndpoint(ClientSession session) {
+ try {
+ session.endpoint().close();
+ } catch (IllegalStateException e) {
+ LOG.debugv("IoT MQTT client {0} was already closed", session.clientId());
+ }
+ }
+
Optional<ConnectionInfo> getConnection(String clientId) {
if (server == null) {
return Optional.empty();
@@ -385,9 +420,11 @@ public class IotMqttBrokerService implements Resettable {
* every client except one that names a custom authorizer, see {@link #authorizeCustom}.
*/
void handleEndpoint(MqttEndpoint endpoint, boolean verifyDevice) {
+ long connectNanos = nanoTime.getAsLong();
String clientId = endpoint.clientIdentifier();
SocketAddress remoteAddress = endpoint.remoteAddress();
String sourceIp = remoteAddress == null ? null : remoteAddress.host();
+ int sourcePort = remoteAddress == null ? -1 : remoteAddress.port();
String principal = null;
if (verifyDevice) {
principal = admittedDevice(endpoint, clientId, sourceIp);
@@ -399,14 +436,18 @@ public class IotMqttBrokerService implements Resettable {
}
} else {
IotCustomAuthorizer.WebSocketUpgrade upgrade = remoteAddress == null ? null : webSocketUpgrades.remove(remoteAddress);
+ if (upgrade != null && upgrade.sourceIp() != null) {
+ sourceIp = upgrade.sourceIp();
+ sourcePort = upgrade.sourcePort();
+ }
String username = endpoint.auth() == null ? null : endpoint.auth().getUsername();
String authorizerName = IotCustomAuthorizer.authorizerName(username, upgrade);
if (authorizerName != null) {
- authorizeCustom(endpoint, authorizerName, upgrade, upgrade == null ? sourceIp : upgrade.sourceIp());
+ authorizeCustom(endpoint, authorizerName, upgrade, sourceIp, sourcePort, connectNanos);
return;
}
}
- admit(endpoint, principal);
+ admit(endpoint, principal, sourceIp, sourcePort, connectNanos);
}
/**
@@ -416,7 +457,7 @@ public class IotMqttBrokerService implements Resettable {
* MQTT 5 with reason code 0x87, as AWS IoT does.
*/
private void authorizeCustom(MqttEndpoint endpoint, String authorizerName, IotCustomAuthorizer.WebSocketUpgrade upgrade,
- String sourceIp) {
+ String sourceIp, int sourcePort, long connectNanos) {
String clientId = endpoint.clientIdentifier();
String username = endpoint.auth() == null ? null : endpoint.auth().getUsername();
String password = endpoint.auth() == null ? null : endpoint.auth().getPassword();
@@ -431,7 +472,7 @@ public class IotMqttBrokerService implements Resettable {
return;
}
if (decision.succeeded() && decision.result()) {
- admit(endpoint, null);
+ admit(endpoint, null, sourceIp, sourcePort, connectNanos);
return;
}
LOG.debugv("IoT MQTT client {0} refused by custom authorizer {1}: {2}", clientId, authorizerName,
@@ -447,39 +488,72 @@ public class IotMqttBrokerService implements Resettable {
});
}
- private void admit(MqttEndpoint endpoint, String principal) {
+ /**
+ * {@code principal} is the certificate id a TLS client was admitted with, null otherwise.
+ * {@code clientIp} and {@code clientPort} are the address the client connected from: the
+ * socket's, or for a WebSocket client the one the bridge recorded at the upgrade, as the
+ * bridge's own socket is loopback. {@code connectNanos} is when the CONNECT arrived, which
+ * the keep-alive runs from.
+ */
+ private void admit(MqttEndpoint endpoint, String principal, String clientIp, int clientPort, long connectNanos) {
String clientId = endpoint.clientIdentifier();
- SocketAddress remoteAddress = endpoint.remoteAddress();
- String sourceIp = remoteAddress == null ? null : remoteAddress.host();
- ClientSession session = new ClientSession(
- clientId,
- endpoint,
- sourceIp,
- remoteAddress == null ? -1 : remoteAddress.port(),
- endpoint.isCleanSession());
-
- endpoint.subscriptionAutoAck(false);
- endpoint.publishAutoAck(false);
- endpoint.exceptionHandler(error -> LOG.warnv("IoT MQTT client {0} error: {1}", clientId, error.getMessage()));
- endpoint.subscribeHandler(message -> handleSubscribe(session, message));
- endpoint.unsubscribeHandler(message -> handleUnsubscribe(session, message));
- endpoint.publishHandler(message -> handlePublish(session, message));
- endpoint.disconnectHandler(ignored -> removeSession(session));
- endpoint.closeHandler(ignored -> removeSession(session));
-
- ClientSession previous = sessionsByClient.put(clientId, session);
- if (previous != null && previous.endpoint() != endpoint) {
- previous.endpoint().close();
- }
-
- endpoint.accept();
+ boolean cleanSession = endpoint.isCleanSession();
+ ClientSession session;
+ ClientSession replaced = null;
+ synchronized (sessionLock) {
+ long versionNumber = presenceVersions.merge(clientId, 0L, (last, unused) -> last + 1);
+ session = new ClientSession(
+ clientId,
+ endpoint,
+ clientIp,
+ clientPort,
+ cleanSession,
+ UUID.randomUUID().toString(),
+ principal,
+ versionNumber,
+ new AtomicBoolean(),
+ new AtomicLong(connectNanos));
+ ClientSession previous = sessionsByClient.put(clientId, session);
+ if (previous != null && previous.endpoint() != endpoint) {
+ endSession(previous, DUPLICATE_CLIENTID);
+ replaced = previous;
+ }
+ }
+ if (replaced != null) {
+ closeEndpoint(replaced);
+ }
+
+ try {
+ endpoint.subscriptionAutoAck(false);
+ endpoint.publishAutoAck(false);
+ endpoint.exceptionHandler(error -> {
+ LOG.warnv("IoT MQTT client {0} error: {1}", clientId, error.getMessage());
+ if (error instanceof TooLongFrameException) {
+ endSession(session, CLIENT_ERROR);
+ endpoint.close();
+ }
+ });
+ endpoint.subscribeHandler(message -> handleSubscribe(session, message));
+ endpoint.unsubscribeHandler(message -> handleUnsubscribe(session, message));
+ endpoint.publishHandler(message -> handlePublish(session, message));
+ endpoint.pingHandler(ignored -> received(session));
+ endpoint.publishAcknowledgeHandler(ignored -> received(session));
+ endpoint.disconnectHandler(ignored -> endSession(session, CLIENT_INITIATED_DISCONNECT));
+ endpoint.closeHandler(ignored -> endSession(session, keepAliveExpired(session) ? MQTT_KEEP_ALIVE_TIMEOUT : CONNECTION_LOST));
+ endpoint.accept();
+ } catch (IllegalStateException e) {
+ // Only another thread closes the endpoint here: a takeover, DeleteConnection or stop,
+ // each of which has already ended the session, so its connected event pairs with that.
+ LOG.debugv("IoT MQTT client {0} closed before it was accepted: {1}", clientId, e.getMessage());
+ }
if (principal != null) {
LOG.debugv("IoT MQTT TLS client {0} admitted as {1}", clientId, principal);
}
+ publishPresence(session, "connected", null);
}
/**
- * The certificate ARN of the device behind the connection when it may connect, otherwise null.
+ * The certificate id of the device behind the connection when it may connect, otherwise null.
* A failure while deciding refuses the client rather than leaving the CONNECT unanswered. A
* refusal is the client's own doing and reaches it as the CONNACK, so it is logged at debug
* only: the CONNECT is reachable by anyone.
@@ -491,12 +565,12 @@ public class IotMqttBrokerService implements Resettable {
LOG.debugv("IoT MQTT TLS client {0} refused: no registered certificate presented", clientId);
return null;
}
- String certificateArn = device.get().certificate().getCertificateArn();
+ String certificateId = device.get().certificate().getCertificateId();
if (!iotService.get().isConnectAllowed(device.get(), clientId, sourceIp, requestedServerName(endpoint.sslSession()))) {
- LOG.debugv("IoT MQTT TLS client {0} refused: iot:Connect is not allowed for {1}", clientId, certificateArn);
+ LOG.debugv("IoT MQTT TLS client {0} refused: iot:Connect is not allowed for {1}", clientId, certificateId);
return null;
}
- return certificateArn;
+ return certificateId;
} catch (RuntimeException e) {
LOG.warnv(e, "IoT MQTT TLS client {0} refused: device verification failed: {1}", clientId, e.getMessage());
return null;
@@ -545,8 +619,7 @@ public class IotMqttBrokerService implements Resettable {
}
private void handleSubscribe(ClientSession session, MqttSubscribeMessage message) {
- Map<String, Subscription> clientSubscriptions = subscriptionsByClient.computeIfAbsent(
- session.clientId(), ignored -> new ConcurrentHashMap<>());
+ received(session);
List<MqttQoS> grantedQos = new ArrayList<>();
List<Subscription> accepted = new ArrayList<>();
@@ -559,38 +632,56 @@ public class IotMqttBrokerService implements Resettable {
}
int granted = qos == MqttQoS.AT_LEAST_ONCE ? 1 : 0;
- Subscription subscription = new Subscription(topicFilter, granted);
- clientSubscriptions.put(topicFilter, subscription);
- accepted.add(subscription);
+ accepted.add(new Subscription(topicFilter, granted));
grantedQos.add(granted == 1 ? MqttQoS.AT_LEAST_ONCE : MqttQoS.AT_MOST_ONCE);
}
+ synchronized (sessionLock) {
+ // An ended session no longer owns its client id's subscriptions; a newer connection may.
+ if (session.ended().get()) {
+ return;
+ }
+ Map<String, Subscription> clientSubscriptions = subscriptionsByClient.computeIfAbsent(
+ session.clientId(), ignored -> new ConcurrentHashMap<>());
+ for (Subscription subscription : accepted) {
+ clientSubscriptions.put(subscription.topicFilter(), subscription);
+ }
+ }
session.endpoint().subscribeAcknowledge(message.messageId(), grantedQos);
deliverRetained(session, accepted);
}
private void handleUnsubscribe(ClientSession session, MqttUnsubscribeMessage message) {
- Map<String, Subscription> clientSubscriptions = subscriptionsByClient.get(session.clientId());
- if (clientSubscriptions != null) {
- for (String topic : message.topics()) {
- clientSubscriptions.remove(topic);
+ received(session);
+ synchronized (sessionLock) {
+ if (session.ended().get()) {
+ return;
}
- if (clientSubscriptions.isEmpty()) {
- subscriptionsByClient.remove(session.clientId(), clientSubscriptions);
+ Map<String, Subscription> clientSubscriptions = subscriptionsByClient.get(session.clientId());
+ if (clientSubscriptions != null) {
+ for (String topic : message.topics()) {
+ clientSubscriptions.remove(topic);
+ }
+ if (clientSubscriptions.isEmpty()) {
+ subscriptionsByClient.remove(session.clientId(), clientSubscriptions);
+ }
}
}
session.endpoint().unsubscribeAcknowledge(message.messageId());
}
private void handlePublish(ClientSession session, MqttPublishMessage message) {
+ received(session);
if (message.payload().length() > MAX_PAYLOAD_SIZE) {
LOG.debugv("IoT MQTT client {0} disconnected: {1} byte payload on {2} exceeds the AWS limit",
session.clientId(), Integer.toString(message.payload().length()), message.topicName());
+ endSession(session, CLIENT_ERROR);
session.endpoint().close();
return;
}
byte[] payload = message.payload().getBytes();
if (message.qosLevel() == MqttQoS.EXACTLY_ONCE) {
+ endSession(session, CLIENT_ERROR);
session.endpoint().close();
return;
}
@@ -619,7 +710,10 @@ public class IotMqttBrokerService implements Resettable {
* its rules.
*/
void evaluateRulesOnWorker(String topic, Runnable rules) {
- long admitted = ruleState.get();
+ evaluateRulesOnWorker(topic, rules, ruleState.get());
+ }
+
+ void evaluateRulesOnWorker(String topic, Runnable rules, long admitted) {
if ((admitted & (RULES_RESETTING | RULES_STOPPED)) != 0) {
return;
}
@@ -688,11 +782,64 @@ public class IotMqttBrokerService implements Resettable {
}
}
- private void removeSession(ClientSession session) {
- sessionsByClient.remove(session.clientId(), session);
- if (session.cleanSession()) {
- subscriptionsByClient.remove(session.clientId());
+ /**
+ * Ends the session once: the first reason reported wins, and only that one publishes the
+ * disconnected event and gets true. The versionNumber counts up only while the session is still
+ * the client id's current one, so a session replaced by a newer connection leaves it alone.
+ */
+ private boolean endSession(ClientSession session, String reason) {
+ synchronized (sessionLock) {
+ if (!session.ended().compareAndSet(false, true)) {
+ return false;
+ }
+ if (sessionsByClient.remove(session.clientId(), session)) {
+ presenceVersions.computeIfPresent(session.clientId(), (id, version) -> version + 1);
+ }
+ if (session.cleanSession()) {
+ subscriptionsByClient.remove(session.clientId());
+ }
+ publishPresence(session, "disconnected", reason);
+ return true;
+ }
+ }
+
+ /**
+ * Publishes the AWS lifecycle event of a session the way a client publish goes: recorded, run
+ * through every region's topic rules on the rule worker, then fanned out to subscribers. A
+ * client id holding {@code #} or {@code +} gets none, as on AWS: its topic would not be valid.
+ */
+ private void publishPresence(ClientSession session, String eventType, String disconnectReason) {
+ if (session.clientId().contains("#") || session.clientId().contains("+")) {
+ return;
}
+ String topic = PRESENCE_TOPIC_PREFIX + eventType + "/" + session.clientId();
+ ObjectNode event = JsonNodeFactory.instance.objectNode();
+ event.put("clientId", session.clientId());
+ event.put("timestamp", System.currentTimeMillis());
+ event.put("eventType", eventType);
+ event.put("sessionIdentifier", session.sessionIdentifier());
+ if (session.principalIdentifier() != null) {
+ event.put("principalIdentifier", session.principalIdentifier());
+ }
+ event.put("versionNumber", session.versionNumber());
+ if (disconnectReason == null) {
+ if (session.sourceIp() != null) {
+ event.put("ipAddress", session.sourceIp());
+ }
+ } else {
+ event.put("clientInitiatedDisconnect", CLIENT_INITIATED_DISCONNECT.equals(disconnectReason));
+ event.put("disconnectReason", disconnectReason);
+ }
+ byte[] payload = event.toString().getBytes(StandardCharsets.UTF_8);
+ // Vert.x runs an endpoint callback holding that endpoint's monitor and the fan-out takes each subscriber's,
+ // so the event is published once the callback returns; events scheduled from one thread keep their order.
+ // Its rules are admitted under the rule state at the time of the event.
+ long admitted = ruleState.get();
+ vertx.runOnContext(ignored -> {
+ iotService.get().publish(topic, payload, false, 0, null, session.clientId(),
+ rules -> evaluateRulesOnWorker(topic, rules, admitted));
+ fanOut(topic, payload, false);
+ });
}
private boolean isValidTopicFilter(String topicFilter) {
@@ -717,6 +864,10 @@ public class IotMqttBrokerService implements Resettable {
return true;
}
String[] filterLevels = topicFilter.split("/", -1);
+ // Measured on AWS IoT: a wildcard first level matches the $aws/things shadow topics but not the lifecycle topics.
+ if (topic.startsWith(PRESENCE_TOPIC_PREFIX) && ("#".equals(filterLevels[0]) || "+".equals(filterLevels[0]))) {
+ return false;
+ }
String[] topicLevels = topic.split("/", -1);
for (int i = 0; i < filterLevels.length; i++) {
String filterLevel = filterLevels[i];
@@ -733,12 +884,29 @@ public class IotMqttBrokerService implements Resettable {
return filterLevels.length == topicLevels.length;
}
+ private void received(ClientSession session) {
+ session.lastPacketNanos().set(nanoTime.getAsLong());
+ }
+
+ private boolean keepAliveExpired(ClientSession session) {
+ int keepAlive = session.endpoint().keepAliveTimeSeconds();
+ // Vert.x closes a client silent for ceil(1.5 * keep-alive) seconds since Netty's last read.
+ // Every packet is stamped here before Netty stamps it, the CONNECT microseconds after.
+ long timeout = TimeUnit.SECONDS.toNanos((long) Math.ceil(keepAlive * 1.5)) - KEEP_ALIVE_SLACK_NANOS;
+ return keepAlive > 0 && nanoTime.getAsLong() - session.lastPacketNanos().get() >= timeout;
+ }
+
private record ClientSession(
String clientId,
MqttEndpoint endpoint,
String sourceIp,
int sourcePort,
- boolean cleanSession) {
+ boolean cleanSession,
+ String sessionIdentifier,
+ String principalIdentifier,
+ long versionNumber,
+ AtomicBoolean ended,
+ AtomicLong lastPacketNanos) {
}
private record Subscription(String topicFilter, int qos) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttWebSocketBridge.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttWebSocketBridge.java
index 8e33a22b9b..5be8d5c871 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttWebSocketBridge.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttWebSocketBridge.java
@@ -120,7 +120,8 @@ public class IotMqttWebSocketBridge {
/**
* The upgrade as AWS hands it to a custom authorizer; the server name only over TLS, from SNI or
- * else the Host header. The client's address rides along for the broker's policy evaluation.
+ * else the Host header. The client's address and port ride along for the broker's policy
+ * evaluation and GetConnection.
*/
private static IotCustomAuthorizer.WebSocketUpgrade upgradeOf(ServerWebSocket ws) {
Map<String, String> headers = new LinkedHashMap<>();
@@ -135,7 +136,7 @@ public class IotMqttWebSocketBridge {
}
SocketAddress client = ws.remoteAddress();
return new IotCustomAuthorizer.WebSocketUpgrade(headers, ws.query() == null ? "" : ws.query(), serverName,
- client == null ? null : client.host());
+ client == null ? null : client.host(), client == null ? -1 : client.port());
}
static final class BridgeSession {
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java
index e50062bb77..710b0d30f7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java
@@ -2035,6 +2035,11 @@ public class IotService {
return false;
}
String[] filterParts = filter.split("/", -1);
+ // Measured on AWS IoT: a wildcard first level matches the $aws/things shadow topics but not the lifecycle topics.
+ if (topic.startsWith(IotMqttBrokerService.PRESENCE_TOPIC_PREFIX)
+ && ("#".equals(filterParts[0]) || "+".equals(filterParts[0]))) {
+ return false;
+ }
String[] topicParts = topic.split("/", -1);
int i = 0;
for (; i < filterParts.length; i++) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotCustomAuthorizerTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotCustomAuthorizerTest.java
index a126a95479..658998f2e7 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotCustomAuthorizerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotCustomAuthorizerTest.java
@@ -389,7 +389,7 @@ class IotCustomAuthorizerTest {
headers.put("sec-websocket-protocol", "mqtt");
List<String> policies = customAuthorizer.connectPolicies("a", "c", "c?" + NAME_PARAM + "a", "secret",
- new WebSocketUpgrade(headers, "", "iot.example.com", null));
+ new WebSocketUpgrade(headers, "", "iot.example.com", null, -1));
assertEquals(List.of(POLICY), policies);
assertEquals("{\"protocolData\":{\"tls\":{\"serverName\":\"iot.example.com\"},\"mqtt\":{\"username\":"
@@ -401,7 +401,7 @@ class IotCustomAuthorizerTest {
@Test
void aConnectWithoutUsernamePasswordOrTlsLeavesThemOut() {
- customAuthorizer.connectPolicies("a", "c", null, null, new WebSocketUpgrade(Map.of(), NAME_PARAM + "a", null, null));
+ customAuthorizer.connectPolicies("a", "c", null, null, new WebSocketUpgrade(Map.of(), NAME_PARAM + "a", null, null, -1));
assertEquals("{\"protocolData\":{\"mqtt\":{\"clientId\":\"c\"},\"http\":{\"headers\":{},"
+ "\"queryString\":\"x-amz-customauthorizer-name=a\"}},\"protocols\":[\"http\",\"mqtt\"],"
@@ -467,11 +467,11 @@ class IotCustomAuthorizerTest {
String signature = sign(key1, "allow-me");
customAuthorizer.connectPolicies("s", "c", "c", null, new WebSocketUpgrade(Map.of(),
- NAME_PARAM + "s&" + SIGNATURE_PARAM + urlEncoded(signature) + "&tok=allow-me", null, null));
+ NAME_PARAM + "s&" + SIGNATURE_PARAM + urlEncoded(signature) + "&tok=allow-me", null, null, -1));
assertEquals("allow-me", events.getLast().path("token").asText());
customAuthorizer.connectPolicies("s", "c", "c", null, new WebSocketUpgrade(
- Map.of("x-amz-customauthorizer-signature", signature, "tok", "allow-me"), "", null, null));
+ Map.of("x-amz-customauthorizer-signature", signature, "tok", "allow-me"), "", null, null, -1));
assertEquals("allow-me", events.getLast().path("token").asText());
}
@@ -489,9 +489,9 @@ class IotCustomAuthorizerTest {
@Test
void theAuthorizerNameComesFromTheUsernameTheWebSocketQueryOrTheUpgradeHeader() {
assertEquals("u", IotCustomAuthorizer.authorizerName("c?" + NAME_PARAM + "u&tok=t", null));
- assertEquals("q", IotCustomAuthorizer.authorizerName("c", new WebSocketUpgrade(Map.of(), NAME_PARAM + "q", null, null)));
+ assertEquals("q", IotCustomAuthorizer.authorizerName("c", new WebSocketUpgrade(Map.of(), NAME_PARAM + "q", null, null, -1)));
assertEquals("h", IotCustomAuthorizer.authorizerName(null,
- new WebSocketUpgrade(Map.of("x-amz-customauthorizer-name", "h"), "", null, null)));
+ new WebSocketUpgrade(Map.of("x-amz-customauthorizer-name", "h"), "", null, null, -1)));
}
@Test
@@ -499,7 +499,7 @@ class IotCustomAuthorizerTest {
assertNull(IotCustomAuthorizer.authorizerName(null, null));
assertNull(IotCustomAuthorizer.authorizerName("plain-user", null));
assertNull(IotCustomAuthorizer.authorizerName("c?X-Amz-Algorithm=AWS4-HMAC-SHA256", new WebSocketUpgrade(
- Map.of("sec-websocket-protocol", "mqtt"), "X-Amz-Signature=00", null, null)));
+ Map.of("sec-websocket-protocol", "mqtt"), "X-Amz-Signature=00", null, null, -1)));
assertEquals("%zz", IotCustomAuthorizer.authorizerName("c?" + NAME_PARAM + "%zz", null),
"a malformed escape is kept as sent instead of failing the CONNECT");
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerDeviceVerificationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerDeviceVerificationTest.java
index de2ece556c..c0a99063d1 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerDeviceVerificationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerDeviceVerificationTest.java
@@ -1,5 +1,7 @@
package io.github.hectorvent.floci.services.iot;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.config.FlociCertificateAuthority;
import io.github.hectorvent.floci.core.common.Pem;
@@ -25,6 +27,7 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
import javax.net.ssl.KeyManager;
import javax.net.ssl.KeyManagerFactory;
@@ -35,6 +38,7 @@ import java.nio.file.Path;
import java.security.KeyStore;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
+import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@@ -50,6 +54,7 @@ import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
@@ -186,6 +191,24 @@ class IotMqttBrokerDeviceVerificationTest {
}
}
+ @Test
+ void anAdmittedDevicesLifecycleEventsCarryItsCertificateIdAsPrincipal() throws Exception {
+ RegisteredDevice device = registered(deviceLeaf);
+ when(service.findRegisteredCertificate(certificate(deviceLeaf))).thenReturn(Optional.of(device));
+ when(service.isConnectAllowed(eq(device), eq("sensor-1"), eq("127.0.0.1"), isNull())).thenReturn(true);
+
+ MqttClient client = connectTls("sensor-1", deviceLeaf, null);
+ client.disconnect();
+ client.close();
+
+ JsonNode connected = publishedEvent("$aws/events/presence/connected/sensor-1");
+ JsonNode disconnected = publishedEvent("$aws/events/presence/disconnected/sensor-1");
+ assertEquals(List.of("clientId", "timestamp", "eventType", "sessionIdentifier", "principalIdentifier",
+ "versionNumber", "ipAddress"), fieldNames(connected));
+ assertEquals(device.certificate().getCertificateId(), connected.get("principalIdentifier").asText());
+ assertEquals(device.certificate().getCertificateId(), disconnected.get("principalIdentifier").asText());
+ }
+
@Test
void aDeniedDeviceIsAnsweredWithConnackNotAuthorized() throws Exception {
RegisteredDevice device = registered(deviceLeaf);
@@ -254,7 +277,8 @@ class IotMqttBrokerDeviceVerificationTest {
client.connect();
assertTrue(client.isConnected());
- verifyNoInteractions(service);
+ verify(service, never()).findRegisteredCertificate(any());
+ verify(service, never()).isConnectAllowed(any(), anyString(), any(), any());
} finally {
client.disconnect();
client.close();
@@ -306,6 +330,19 @@ class IotMqttBrokerDeviceVerificationTest {
assertFalse(refused.getMessage().isBlank());
}
+ /** The payload the broker handed the IoT service for the one publish on {@code topic}, an MQTT client's publish. */
+ private JsonNode publishedEvent(String topic) throws IOException {
+ ArgumentCaptor<byte[]> payload = ArgumentCaptor.forClass(byte[].class);
+ verify(service, timeout(10_000)).publish(eq(topic), payload.capture(), eq(false), eq(0), isNull(), eq("sensor-1"), any());
+ return new ObjectMapper().readTree(payload.getValue());
+ }
+
+ private static List<String> fieldNames(JsonNode node) {
+ List<String> names = new ArrayList<>();
+ node.fieldNames().forEachRemaining(names::add);
+ return names;
+ }
+
private static RegisteredDevice registered(CertificateGenerator.GeneratedCertificate leaf) {
IotCertificate certificate = new IotCertificate();
certificate.setCertificateId("id-" + leaf.hashCode());
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerServiceTest.java
index 1aedc3da97..e275e76cbe 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerServiceTest.java
@@ -22,12 +22,14 @@ import io.github.hectorvent.floci.services.lambda.LambdaService;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.sns.SnsService;
import io.github.hectorvent.floci.services.sqs.SqsService;
+import io.github.hectorvent.floci.testing.ThreadStates;
import io.github.hectorvent.floci.testutil.FreePorts;
import io.netty.handler.codec.mqtt.MqttQoS;
import io.quarkus.tls.CertificateUpdatedEvent;
import io.quarkus.tls.TlsConfiguration;
import io.quarkus.tls.TlsConfigurationRegistry;
import io.quarkus.tls.runtime.config.TlsConfig;
+import io.vertx.core.Context;
import io.vertx.core.Handler;
import io.vertx.core.Vertx;
import io.vertx.core.buffer.Buffer;
@@ -36,11 +38,16 @@ import io.vertx.core.net.PemKeyCertOptions;
import io.vertx.core.net.SocketAddress;
import io.vertx.mqtt.MqttEndpoint;
import io.vertx.mqtt.MqttTopicSubscription;
+import io.vertx.mqtt.messages.MqttPublishMessage;
import io.vertx.mqtt.messages.MqttSubscribeMessage;
+import io.vertx.mqtt.messages.MqttUnsubscribeMessage;
import jakarta.enterprise.inject.Instance;
+import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
+import org.eclipse.paho.client.mqttv3.MqttCallback;
import org.eclipse.paho.client.mqttv3.MqttClient;
import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
import org.eclipse.paho.client.mqttv3.MqttException;
+import org.eclipse.paho.client.mqttv3.MqttMessage;
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
@@ -48,6 +55,8 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.mockito.AdditionalAnswers;
+import org.mockito.ArgumentCaptor;
import javax.net.ssl.KeyManager;
import javax.net.ssl.KeyManagerFactory;
@@ -57,7 +66,9 @@ import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509ExtendedKeyManager;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
@@ -68,14 +79,17 @@ import java.security.Principal;
import java.security.PrivateKey;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -83,20 +97,29 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantLock;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
@@ -105,11 +128,12 @@ import static org.mockito.Mockito.when;
* The MQTT over TLS listener against a real Vert.x and a mocked TLS registry: what it serves,
* when it is not opened, how a bind failure is reported, and how a reloaded server certificate
* reaches the next handshake without a restart. Also that a publish's topic rules run on the
- * broker's own single-thread worker, off the event loop.
+ * broker's own single-thread worker, off the event loop, and the AWS lifecycle events of a session.
*/
class IotMqttBrokerServiceTest {
private static final CertificateGenerator GENERATOR = new CertificateGenerator();
+ private static final ObjectMapper MAPPER = new ObjectMapper();
/**
* AWS IoT's default security policy, IoTSecurityPolicy_TLS13_1_2_2022_10, in JSSE names: the
@@ -135,6 +159,9 @@ class IotMqttBrokerServiceTest {
@SuppressWarnings("unchecked")
private final Instance<IotService> iotService = mock(Instance.class);
private final IotCustomAuthorizer customAuthorizer = mock(IotCustomAuthorizer.class);
+ private final IotPublishEventRecorder recorder = new IotPublishEventRecorder();
+ /** The broker's clock, which a keep-alive test moves forward instead of sleeping. */
+ private final AtomicLong nanoTime = new AtomicLong();
private int plainPort;
private int tlsPort;
private IotMqttBrokerService broker;
@@ -166,7 +193,7 @@ class IotMqttBrokerServiceTest {
when(config.tls().enabled()).thenReturn(true);
when(registry.getDefault()).thenReturn(Optional.of(tls));
when(tls.getKeyStoreOptions()).thenReturn(pem(bootLeaf));
- broker = new IotMqttBrokerService(config, vertx, iotService, customAuthorizer, registry);
+ broker = new IotMqttBrokerService(config, vertx, iotService, customAuthorizer, registry, nanoTime::get);
}
@AfterEach
@@ -498,7 +525,7 @@ class IotMqttBrokerServiceTest {
.thenReturn(List.of("{}"));
SocketAddress bridge = SocketAddress.inetSocketAddress(50123, "127.0.0.1");
broker.webSocketUpgrades().put(bridge, new IotCustomAuthorizer.WebSocketUpgrade(
- Map.of(), IotCustomAuthorizer.NAME_PARAM + "=ws-auth", null, "203.0.113.7"));
+ Map.of(), IotCustomAuthorizer.NAME_PARAM + "=ws-auth", null, "203.0.113.7", 41233));
MqttEndpoint endpoint = mock(MqttEndpoint.class);
when(endpoint.clientIdentifier()).thenReturn("ws-client");
when(endpoint.remoteAddress()).thenReturn(bridge);
@@ -508,6 +535,656 @@ class IotMqttBrokerServiceTest {
verify(service, timeout(10_000)).isConnectAllowedBy(List.of("{}"), "ws-client", "203.0.113.7", null);
}
+ @Test
+ void aWebSocketClientsConnectedEventCarriesTheWebSocketClientsAddress() throws Exception {
+ startBrokerRecordingPublishes();
+ SocketAddress bridge = SocketAddress.inetSocketAddress(50124, "127.0.0.1");
+ broker.webSocketUpgrades().put(bridge, new IotCustomAuthorizer.WebSocketUpgrade(Map.of(), "", null, "203.0.113.7", 41234));
+ MqttEndpoint endpoint = mock(MqttEndpoint.class);
+ when(endpoint.clientIdentifier()).thenReturn("ws-client");
+ when(endpoint.remoteAddress()).thenReturn(bridge);
+ when(endpoint.isConnected()).thenReturn(true);
+
+ broker.handleEndpoint(endpoint, false);
+
+ assertEquals("203.0.113.7", awaitPresence("connected", "ws-client", 1).get(0).get("ipAddress").asText(),
+ "the WebSocket client's address, not the bridge's loopback socket");
+ assertEquals(new IotMqttBrokerService.ConnectionInfo("ws-client", "203.0.113.7", 41234),
+ broker.getConnection("ws-client").orElseThrow(), "the WebSocket client's address and port");
+ }
+
+ @Test
+ void aConnectPublishesTheConnectedEventInAwsFieldOrder() throws Exception {
+ startBrokerRecordingPublishes();
+ long before = System.currentTimeMillis();
+ MqttClient client = connectPlain("presence-client");
+ try {
+ JsonNode connected = awaitPresence("connected", "presence-client", 1).get(0);
+
+ assertEquals(List.of("clientId", "timestamp", "eventType", "sessionIdentifier", "versionNumber", "ipAddress"),
+ fieldNames(connected), "no principalIdentifier on the plaintext listener");
+ assertEquals("presence-client", connected.get("clientId").asText());
+ assertTrue(connected.get("timestamp").isIntegralNumber(), connected.toString());
+ assertTrue(connected.get("timestamp").asLong() >= before
+ && connected.get("timestamp").asLong() <= System.currentTimeMillis(), "epoch milliseconds: " + connected);
+ assertEquals("connected", connected.get("eventType").asText());
+ assertDashedUuid(connected.get("sessionIdentifier").asText());
+ assertTrue(connected.get("versionNumber").isIntegralNumber(), connected.toString());
+ assertEquals(0, connected.get("versionNumber").asLong(), "a fresh client id starts at 0");
+ assertEquals("127.0.0.1", connected.get("ipAddress").asText());
+ } finally {
+ disconnect(client);
+ }
+ }
+
+ @Test
+ void aCleanDisconnectEndsTheSameSessionAndAReconnectCountsUpByTwo() throws Exception {
+ startBrokerRecordingPublishes();
+ disconnect(connectPlain("presence-client"));
+
+ JsonNode connected = awaitPresence("connected", "presence-client", 1).get(0);
+ JsonNode disconnected = awaitPresence("disconnected", "presence-client", 1).get(0);
+ assertEquals(List.of("clientId", "timestamp", "eventType", "sessionIdentifier", "versionNumber",
+ "clientInitiatedDisconnect", "disconnectReason"), fieldNames(disconnected), "no ipAddress on disconnected");
+ assertEquals("presence-client", disconnected.get("clientId").asText());
+ assertTrue(disconnected.get("timestamp").isIntegralNumber(), disconnected.toString());
+ assertEquals("disconnected", disconnected.get("eventType").asText());
+ assertEquals(connected.get("sessionIdentifier"), disconnected.get("sessionIdentifier"));
+ assertEquals(0, disconnected.get("versionNumber").asLong());
+ assertTrue(disconnected.get("clientInitiatedDisconnect").isBoolean(), disconnected.toString());
+ assertTrue(disconnected.get("clientInitiatedDisconnect").asBoolean());
+ assertEquals("CLIENT_INITIATED_DISCONNECT", disconnected.get("disconnectReason").asText());
+
+ disconnect(connectPlain("presence-client"));
+
+ JsonNode reconnected = awaitPresence("connected", "presence-client", 2).get(1);
+ assertEquals(2, reconnected.get("versionNumber").asLong(), "a reconnect after a session ended adds 2");
+ assertNotEquals(connected.get("sessionIdentifier"), reconnected.get("sessionIdentifier"));
+ JsonNode redisconnected = awaitPresence("disconnected", "presence-client", 2).get(1);
+ assertEquals(reconnected.get("sessionIdentifier"), redisconnected.get("sessionIdentifier"));
+ assertEquals(2, redisconnected.get("versionNumber").asLong());
+ }
+
+ @Test
+ void aConnectionClosedWithoutDisconnectIsReportedAsConnectionLost() throws Exception {
+ startBrokerRecordingPublishes();
+ try (Socket socket = rawConnect("abrupt")) {
+ awaitPresence("connected", "abrupt", 1);
+ }
+
+ JsonNode disconnected = awaitPresence("disconnected", "abrupt", 1).get(0);
+ assertEquals("CONNECTION_LOST", disconnected.get("disconnectReason").asText());
+ assertTrue(disconnected.get("clientInitiatedDisconnect").isBoolean(), disconnected.toString());
+ assertFalse(disconnected.get("clientInitiatedDisconnect").asBoolean());
+ assertEquals(0, disconnected.get("versionNumber").asLong());
+ }
+
+ @Test
+ void aSecondConnectWithTheSameClientIdEndsTheFirstSessionAsADuplicateBeforeTheNewOneConnects() throws Exception {
+ startBrokerRecordingPublishes();
+ MqttClient first = connectPlain("shared");
+ MqttClient second = null;
+ try {
+ awaitPresence("connected", "shared", 1);
+ second = connectPlain("shared");
+ awaitPresence("connected", "shared", 2);
+ awaitDisconnected(first);
+ barrier();
+
+ List<JsonNode> events = presenceOf("shared");
+ assertEquals(List.of("connected", "disconnected", "connected"), eventTypes(events),
+ "the replaced session's own close publishes nothing more: " + events);
+ JsonNode replaced = events.get(1);
+ assertEquals("DUPLICATE_CLIENTID", replaced.get("disconnectReason").asText());
+ assertFalse(replaced.get("clientInitiatedDisconnect").asBoolean());
+ assertEquals(events.get(0).get("sessionIdentifier"), replaced.get("sessionIdentifier"));
+ assertEquals(0, replaced.get("versionNumber").asLong());
+ assertEquals(1, events.get(2).get("versionNumber").asLong(), "a takeover adds 1");
+ assertNotEquals(events.get(0).get("sessionIdentifier"), events.get(2).get("sessionIdentifier"));
+ } finally {
+ disconnect(first);
+ if (second != null) {
+ disconnect(second);
+ }
+ }
+ }
+
+ @Test
+ void deleteConnectionReportsOneApiInitiatedDisconnect() throws Exception {
+ startBrokerRecordingPublishes();
+ MqttClient client = connectPlain("deleted");
+ try {
+ awaitPresence("connected", "deleted", 1);
+
+ assertTrue(broker.disconnectClient("deleted", false));
+ awaitPresence("disconnected", "deleted", 1);
+ awaitDisconnected(client);
+ barrier();
+
+ List<JsonNode> disconnected = presence("disconnected", "deleted");
+ assertEquals(1, disconnected.size(), disconnected.toString());
+ assertEquals("API_INITIATED_DISCONNECT", disconnected.get(0).get("disconnectReason").asText());
+ assertFalse(disconnected.get(0).get("clientInitiatedDisconnect").asBoolean());
+ } finally {
+ disconnect(client);
+ }
+ }
+
+ /**
+ * A DeleteConnection that finds the session after another ending won it, here a broker stop
+ * still closing the endpoint, reports the client as not connected instead of closing the
+ * endpoint again.
+ */
+ @Test
+ void deleteConnectionLosingTheSessionToAnotherEndingReportsNotConnected() throws Exception {
+ startBrokerRecordingPublishes();
+ CountDownLatch closing = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicBoolean closed = new AtomicBoolean();
+ MqttEndpoint endpoint = mock(MqttEndpoint.class);
+ when(endpoint.clientIdentifier()).thenReturn("racing");
+ doAnswer(invocation -> {
+ if (!closed.compareAndSet(false, true)) {
+ throw new IllegalStateException("MQTT endpoint is closed");
+ }
+ closing.countDown();
+ release.await(10, TimeUnit.SECONDS);
+ return null;
+ }).when(endpoint).close();
+ broker.handleEndpoint(endpoint, false);
+ ExecutorService stopping = Executors.newSingleThreadExecutor();
+ try {
+ Future<?> stop = stopping.submit(broker::stop);
+ assertTrue(closing.await(10, TimeUnit.SECONDS), "the stop is closing the endpoint");
+
+ assertFalse(broker.disconnectClient("racing", false));
+
+ release.countDown();
+ stop.get(10, TimeUnit.SECONDS);
+ } finally {
+ release.countDown();
+ stopping.shutdown();
+ }
+ }
+
+ /**
+ * A DeleteConnection that wins the session can still find the endpoint closed: the client's
+ * DISCONNECT, arriving in between, loses the session, yet Vert.x closes the endpoint after it.
+ */
+ @Test
+ void deleteConnectionOfAnEndpointAlreadyClosedByTheClientStillSucceeds() throws Exception {
+ startBrokerRecordingPublishes();
+ MqttEndpoint endpoint = mock(MqttEndpoint.class);
+ when(endpoint.clientIdentifier()).thenReturn("closed-by-client");
+ doThrow(new IllegalStateException("MQTT endpoint is closed")).when(endpoint).close();
+ broker.handleEndpoint(endpoint, false);
+
+ assertTrue(broker.disconnectClient("closed-by-client", false));
+
+ List<JsonNode> disconnected = awaitPresence("disconnected", "closed-by-client", 1);
+ assertEquals(1, disconnected.size(), disconnected.toString());
+ assertEquals("API_INITIATED_DISCONNECT", disconnected.get(0).get("disconnectReason").asText());
+ }
+
+ /** A takeover can find the previous endpoint already closed by its own client, and still admits the new one. */
+ @Test
+ void aTakeoverOfAnEndpointAlreadyClosedStillAcceptsTheNewConnection() throws Exception {
+ startBrokerRecordingPublishes();
+ MqttEndpoint previous = mock(MqttEndpoint.class);
+ when(previous.clientIdentifier()).thenReturn("closed-takeover");
+ doThrow(new IllegalStateException("MQTT endpoint is closed")).when(previous).close();
+ broker.handleEndpoint(previous, false);
+ MqttEndpoint replacement = mock(MqttEndpoint.class);
+ when(replacement.clientIdentifier()).thenReturn("closed-takeover");
+
+ assertDoesNotThrow(() -> broker.handleEndpoint(replacement, false));
+
+ verify(replacement).accept();
+ awaitPresence("connected", "closed-takeover", 2);
+ List<JsonNode> events = presenceOf("closed-takeover");
+ assertEquals(List.of("connected", "disconnected", "connected"), eventTypes(events), events.toString());
+ assertEquals("DUPLICATE_CLIENTID", events.get(1).get("disconnectReason").asText());
+ }
+
+ /**
+ * Two CONNECTs for one client id admitted at once on two event loops: the first is held after it
+ * took its versionNumber while the second completes. The session left current is the one with
+ * the highest versionNumber, so DeleteConnection ends that one. The first finds its endpoint
+ * closed by the takeover, as Vert.x refuses to configure a closed endpoint, and its connected
+ * and DUPLICATE_CLIENTID disconnected events still pair up.
+ */
+ @Test
+ void overlappingConnectsForOneClientIdLeaveTheNewestVersionCurrent() throws Exception {
+ startBrokerRecordingPublishes();
+ CountDownLatch admitting = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicBoolean firstClosed = new AtomicBoolean();
+ MqttEndpoint first = mock(MqttEndpoint.class);
+ when(first.clientIdentifier()).thenReturn("overlap");
+ doAnswer(invocation -> {
+ firstClosed.set(true);
+ return null;
+ }).when(first).close();
+ doAnswer(invocation -> {
+ admitting.countDown();
+ release.await(10, TimeUnit.SECONDS);
+ if (firstClosed.get()) {
+ throw new IllegalStateException("MQTT endpoint is closed");
+ }
+ return null;
+ }).when(first).subscriptionAutoAck(false);
+ MqttEndpoint second = mock(MqttEndpoint.class);
+ when(second.clientIdentifier()).thenReturn("overlap");
+ ExecutorService firstEventLoop = Executors.newSingleThreadExecutor();
+ try {
+ Future<?> firstAdmit = firstEventLoop.submit(() -> broker.handleEndpoint(first, false));
+ assertTrue(admitting.await(10, TimeUnit.SECONDS), "the first CONNECT is being admitted");
+ broker.handleEndpoint(second, false);
+ release.countDown();
+ firstAdmit.get(10, TimeUnit.SECONDS);
+ } finally {
+ release.countDown();
+ firstEventLoop.shutdown();
+ }
+
+ List<JsonNode> connected = awaitPresence("connected", "overlap", 2);
+ assertTrue(broker.disconnectClient("overlap", false));
+
+ long newest = Math.max(connected.get(0).get("versionNumber").asLong(), connected.get(1).get("versionNumber").asLong());
+ assertNotEquals(connected.get(0).get("versionNumber"), connected.get(1).get("versionNumber"), connected.toString());
+ JsonNode deleted = awaitPresence("disconnected", "overlap", 2).stream()
+ .filter(event -> "API_INITIATED_DISCONNECT".equals(event.get("disconnectReason").asText()))
+ .findFirst()
+ .orElseThrow();
+ assertEquals(newest, deleted.get("versionNumber").asLong(), "the session left current: " + presenceOf("overlap"));
+ long overlapped = newest == connected.get(0).get("versionNumber").asLong()
+ ? connected.get(1).get("versionNumber").asLong()
+ : connected.get(0).get("versionNumber").asLong();
+ assertEquals(List.of("connected", "disconnected"), eventTypes(presenceOf("overlap").stream()
+ .filter(event -> event.get("versionNumber").asLong() == overlapped)
+ .toList()).stream().sorted().toList(), presenceOf("overlap").toString());
+ }
+
+ /**
+ * A DeleteConnection with cleanSession racing a reconnect of the client id it ends: the reconnect
+ * waits until DeleteConnection is done with the old session, so the subscriptions the new
+ * session makes stay. The DeleteConnection is held where it schedules the disconnected event.
+ */
+ @Test
+ void deleteConnectionLeavesTheSubscriptionsOfAReconnectRacingIt() throws Exception {
+ IotService service = mock(IotService.class);
+ when(iotService.get()).thenReturn(service);
+ when(service.listRetainedMessages(null, null)).thenReturn(new IotService.Page<>(List.of(), null));
+ CountDownLatch ending = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicReference<Thread> deletingThread = new AtomicReference<>();
+ Vertx heldVertx = mock(Vertx.class, AdditionalAnswers.delegatesTo(vertx));
+ doAnswer(invocation -> {
+ if (Thread.currentThread() == deletingThread.get()) {
+ ending.countDown();
+ release.await(10, TimeUnit.SECONDS);
+ }
+ vertx.runOnContext(invocation.getArgument(0));
+ return null;
+ }).when(heldVertx).runOnContext(any());
+ IotMqttBrokerService racing = new IotMqttBrokerService(config, heldVertx, iotService, customAuthorizer, registry);
+ MqttEndpoint old = mock(MqttEndpoint.class);
+ when(old.clientIdentifier()).thenReturn("reconnecting");
+ racing.handleEndpoint(old, false);
+ MqttEndpoint reconnect = mock(MqttEndpoint.class);
+ when(reconnect.clientIdentifier()).thenReturn("reconnecting");
+ AtomicReference<Handler<MqttSubscribeMessage>> subscribeHandler = new AtomicReference<>();
+ when(reconnect.subscribeHandler(any())).thenAnswer(invocation -> {
+ subscribeHandler.set(invocation.getArgument(0));
+ return reconnect;
+ });
+ MqttTopicSubscription requested = mock(MqttTopicSubscription.class);
+ when(requested.topicName()).thenReturn("devices/reconnecting/commands");
+ when(requested.qualityOfService()).thenReturn(MqttQoS.AT_MOST_ONCE);
+ MqttSubscribeMessage subscribe = mock(MqttSubscribeMessage.class);
+ when(subscribe.topicSubscriptions()).thenReturn(List.of(requested));
+ AtomicReference<Thread> reconnectingThread = new AtomicReference<>();
+ ExecutorService deleting = Executors.newSingleThreadExecutor();
+ ExecutorService reconnecting = Executors.newSingleThreadExecutor();
+ try {
+ Future<Boolean> delete = deleting.submit(() -> {
+ deletingThread.set(Thread.currentThread());
+ return racing.disconnectClient("reconnecting", true);
+ });
+ assertTrue(ending.await(10, TimeUnit.SECONDS), "DeleteConnection is ending the old session");
+ Future<?> reconnected = reconnecting.submit(() -> {
+ reconnectingThread.set(Thread.currentThread());
+ racing.handleEndpoint(reconnect, false);
+ subscribeHandler.get().handle(subscribe);
+ });
+ ThreadStates.awaitBlockedOrDone(reconnectingThread, reconnected);
+ release.countDown();
+
+ assertTrue(delete.get(10, TimeUnit.SECONDS));
+ reconnected.get(10, TimeUnit.SECONDS);
+ } finally {
+ release.countDown();
+ deleting.shutdown();
+ reconnecting.shutdown();
+ }
+ assertEquals(List.of("devices/reconnecting/commands"), racing.listSubscriptions("reconnecting"));
+ }
+
+ @Test
+ void aPayloadAboveTheAwsLimitEndsTheSessionWithAClientError() throws Exception {
+ startBrokerRecordingPublishes();
+ try (Socket socket = rawConnect("oversized")) {
+ rawPublish(socket, 0, "devices/d1/big", new byte[128 * 1024 + 1]);
+
+ awaitPresence("disconnected", "oversized", 1);
+ }
+ barrier();
+
+ List<JsonNode> disconnected = presence("disconnected", "oversized");
+ assertEquals(1, disconnected.size(), disconnected.toString());
+ assertEquals("CLIENT_ERROR", disconnected.get(0).get("disconnectReason").asText());
+ assertFalse(disconnected.get(0).get("clientInitiatedDisconnect").asBoolean());
+ }
+
+ @Test
+ void aQos2PublishEndsTheSessionWithAClientError() throws Exception {
+ startBrokerRecordingPublishes();
+ try (Socket socket = rawConnect("qos2")) {
+ rawPublish(socket, 2, "devices/d1/metrics", "{}".getBytes(StandardCharsets.UTF_8));
+
+ awaitPresence("disconnected", "qos2", 1);
+ }
+ barrier();
+
+ List<JsonNode> disconnected = presence("disconnected", "qos2");
+ assertEquals(1, disconnected.size(), disconnected.toString());
+ assertEquals("CLIENT_ERROR", disconnected.get(0).get("disconnectReason").asText());
+ }
+
+ /** A PUBLISH beyond the decoder's packet limit never reaches the publish handler, yet AWS reports the same client error. */
+ @Test
+ void aPacketAboveTheDecoderLimitEndsTheSessionWithAClientError() throws Exception {
+ startBrokerRecordingPublishes();
+ try (Socket socket = rawConnect("undecodable")) {
+ rawPublish(socket, 0, "devices/d1/huge", new byte[150 * 1024]);
+
+ awaitPresence("disconnected", "undecodable", 1);
+ }
+ barrier();
+
+ List<JsonNode> disconnected = presence("disconnected", "undecodable");
+ assertEquals(1, disconnected.size(), disconnected.toString());
+ assertEquals("CLIENT_ERROR", disconnected.get(0).get("disconnectReason").asText());
+ assertFalse(disconnected.get(0).get("clientInitiatedDisconnect").asBoolean());
+ }
+
+ @Test
+ void aClientIdHoldingAWildcardGetsNoLifecycleEvents() throws Exception {
+ startBrokerRecordingPublishes();
+ try (Socket plus = rawConnect("bad+id"); Socket hash = rawConnect("bad#id")) {
+ assertTrue(broker.getConnection("bad+id").isPresent(), "the client is admitted");
+ assertTrue(broker.getConnection("bad#id").isPresent(), "the client is admitted");
+ }
+ barrier();
+
+ assertEquals(List.of(), recorder.recentEvents().stream()
+ .map(IotPublishEvent::topic)
+ .filter(topic -> topic.contains("bad"))
+ .toList());
+ }
+
+ /**
+ * As AWS IoT does for lifecycle topics: a filter whose first level is a wildcard does not
+ * match them. A later ordinary publish both wildcard subscribers match shows they got nothing
+ * before it.
+ */
+ @Test
+ void onlyASubscriptionWithALiteralFirstLevelReceivesLifecycleEvents() throws Exception {
+ startBrokerRecordingPublishes();
+ BlockingQueue<String> catchAll = new LinkedBlockingQueue<>();
+ BlockingQueue<String> wildcardFirst = new LinkedBlockingQueue<>();
+ BlockingQueue<String> literal = new LinkedBlockingQueue<>();
+ MqttClient catchAllSubscriber = subscribedPlain("catch-all", "#", catchAll);
+ MqttClient wildcardSubscriber = subscribedPlain("wildcard-first", "+/events/presence/+/+", wildcardFirst);
+ MqttClient literalSubscriber = subscribedPlain("literal", "$aws/events/presence/connected/+", literal);
+ try {
+ MqttClient watched = connectPlain("watched");
+ try {
+ assertEquals("$aws/events/presence/connected/watched", literal.poll(10, TimeUnit.SECONDS));
+ watched.publish("barrier/events/presence/connected/watched", new byte[0], 1, false);
+ } finally {
+ disconnect(watched);
+ }
+
+ assertEquals("barrier/events/presence/connected/watched", catchAll.poll(10, TimeUnit.SECONDS),
+ "no lifecycle event reached the # subscriber");
+ assertEquals("barrier/events/presence/connected/watched", wildcardFirst.poll(10, TimeUnit.SECONDS),
+ "no lifecycle event reached the +/events/presence/+/+ subscriber");
+ } finally {
+ disconnect(catchAllSubscriber);
+ disconnect(wildcardSubscriber);
+ disconnect(literalSubscriber);
+ }
+ }
+
+ /** As measured on AWS IoT, a {@code #} subscriber receives the shadow response topics, unlike the lifecycle ones. */
+ @Test
+ void aSubscriptionOnHashReceivesShadowResponseTopics() throws Exception {
+ startBrokerRecordingPublishes();
+ BlockingQueue<String> topics = new LinkedBlockingQueue<>();
+ MqttClient subscriber = subscribedPlain("shadow-watcher", "#", topics);
+ try {
+ broker.publish("$aws/things/thermostat/shadow/update/accepted", "{}".getBytes(StandardCharsets.UTF_8));
+
+ assertEquals("$aws/things/thermostat/shadow/update/accepted", topics.poll(10, TimeUnit.SECONDS));
+ } finally {
+ disconnect(subscriber);
+ }
+ }
+
+ @Test
+ void aBrokerStopPublishesNoDisconnectedEvent() throws Exception {
+ startBrokerRecordingPublishes();
+ MqttClient client = connectPlain("stopped");
+ try {
+ awaitPresence("connected", "stopped", 1);
+
+ broker.stop();
+ awaitDisconnected(client);
+ awaitClosed(plainPort);
+ awaitClosed(tlsPort);
+ broker.startIfEnabled();
+ barrier();
+
+ assertEquals(List.of(), presence("disconnected", "stopped"));
+ } finally {
+ disconnect(client);
+ }
+ }
+
+ /**
+ * Vert.x runs an endpoint's disconnect callback holding that endpoint's monitor, and a
+ * subscriber's isConnected and publish take the subscriber's own. With the subscriber's monitor
+ * held, as by its event loop fanning out to the departing client, the callback still returns:
+ * the disconnected event reaches the subscriber only afterwards, so the two cannot deadlock.
+ */
+ @Test
+ void aDisconnectCallbackReturnsWhileASubscribersMonitorIsHeld() throws Exception {
+ startBrokerRecordingPublishes();
+ ReentrantLock subscriberMonitor = new ReentrantLock();
+ MqttEndpoint subscriber = mock(MqttEndpoint.class);
+ when(subscriber.clientIdentifier()).thenReturn("busy-subscriber");
+ when(subscriber.isConnected()).thenAnswer(invocation -> {
+ subscriberMonitor.lock();
+ try {
+ return true;
+ } finally {
+ subscriberMonitor.unlock();
+ }
+ });
+ AtomicReference<Handler<MqttSubscribeMessage>> subscribeHandler = new AtomicReference<>();
+ when(subscriber.subscribeHandler(any())).thenAnswer(invocation -> {
+ subscribeHandler.set(invocation.getArgument(0));
+ return subscriber;
+ });
+ MqttTopicSubscription requested = mock(MqttTopicSubscription.class);
+ when(requested.topicName()).thenReturn("$aws/events/presence/+/departing");
+ when(requested.qualityOfService()).thenReturn(MqttQoS.AT_MOST_ONCE);
+ MqttSubscribeMessage subscribe = mock(MqttSubscribeMessage.class);
+ when(subscribe.topicSubscriptions()).thenReturn(List.of(requested));
+ broker.handleEndpoint(subscriber, false);
+ subscribeHandler.get().handle(subscribe);
+ MqttEndpoint departing = mock(MqttEndpoint.class);
+ when(departing.clientIdentifier()).thenReturn("departing");
+ AtomicReference<Handler<Void>> disconnectHandler = new AtomicReference<>();
+ when(departing.disconnectHandler(any())).thenAnswer(invocation -> {
+ disconnectHandler.set(invocation.getArgument(0));
+ return departing;
+ });
+ broker.handleEndpoint(departing, false);
+ verify(subscriber, timeout(10_000))
+ .publish(eq("$aws/events/presence/connected/departing"), any(), any(), anyBoolean(), anyBoolean());
+
+ ExecutorService departingEventLoop = Executors.newSingleThreadExecutor();
+ subscriberMonitor.lock();
+ try {
+ Future<?> callback = departingEventLoop.submit(() -> disconnectHandler.get().handle(null));
+
+ assertDoesNotThrow(() -> callback.get(5, TimeUnit.SECONDS),
+ "the disconnect callback returned while the subscriber's monitor is held");
+ } finally {
+ subscriberMonitor.unlock();
+ departingEventLoop.shutdown();
+ }
+ verify(subscriber, timeout(10_000))
+ .publish(eq("$aws/events/presence/disconnected/departing"), any(), any(), anyBoolean(), anyBoolean());
+ }
+
+ /** A PUBLISH, SUBSCRIBE, UNSUBSCRIBE or PUBACK counts as traffic for the keep-alive, as a PINGREQ does. */
+ @Test
+ void anAbruptCloseAfterAPublishSubscribeUnsubscribeOrPubackIsAConnectionLossNotAKeepAliveTimeout() throws Exception {
+ startBrokerRecordingPublishes();
+ MqttEndpoint publishing = keepAliveEndpoint("publishing");
+ MqttEndpoint subscribing = keepAliveEndpoint("subscribing");
+ MqttEndpoint unsubscribing = keepAliveEndpoint("unsubscribing");
+ MqttEndpoint acknowledging = keepAliveEndpoint("acknowledging");
+ nanoTime.addAndGet(Duration.ofSeconds(2).toNanos());
+
+ ArgumentCaptor<Handler<MqttPublishMessage>> publish = handlerCaptor();
+ verify(publishing).publishHandler(publish.capture());
+ MqttPublishMessage message = mock(MqttPublishMessage.class);
+ when(message.payload()).thenReturn(Buffer.buffer("{}"));
+ when(message.qosLevel()).thenReturn(MqttQoS.AT_MOST_ONCE);
+ when(message.topicName()).thenReturn("keepalive/publishing");
+ publish.getValue().handle(message);
+ ArgumentCaptor<Handler<MqttSubscribeMessage>> subscribe = handlerCaptor();
+ verify(subscribing).subscribeHandler(subscribe.capture());
+ MqttTopicSubscription requested = mock(MqttTopicSubscription.class);
+ when(requested.topicName()).thenReturn("keepalive/subscribing");
+ when(requested.qualityOfService()).thenReturn(MqttQoS.AT_MOST_ONCE);
+ MqttSubscribeMessage subscription = mock(MqttSubscribeMessage.class);
+ when(subscription.topicSubscriptions()).thenReturn(List.of(requested));
+ subscribe.getValue().handle(subscription);
+ ArgumentCaptor<Handler<MqttUnsubscribeMessage>> unsubscribe = handlerCaptor();
+ verify(unsubscribing).unsubscribeHandler(unsubscribe.capture());
+ MqttUnsubscribeMessage unsubscription = mock(MqttUnsubscribeMessage.class);
+ when(unsubscription.topics()).thenReturn(List.of("keepalive/unsubscribing"));
+ unsubscribe.getValue().handle(unsubscription);
+ ArgumentCaptor<Handler<Integer>> puback = handlerCaptor();
+ verify(acknowledging).publishAcknowledgeHandler(puback.capture());
+ puback.getValue().handle(1);
+ for (MqttEndpoint endpoint : List.of(publishing, subscribing, unsubscribing, acknowledging)) {
+ ArgumentCaptor<Handler<Void>> close = handlerCaptor();
+ verify(endpoint).closeHandler(close.capture());
+ close.getValue().handle(null);
+ }
+
+ for (String clientId : List.of("publishing", "subscribing", "unsubscribing", "acknowledging")) {
+ assertEquals("CONNECTION_LOST",
+ awaitPresence("disconnected", clientId, 1).get(0).get("disconnectReason").asText(), clientId);
+ }
+ }
+
+ /** Vert.x times a client out after 1.5 times its keep-alive, so a socket lost before then is a connection loss. */
+ @Test
+ void anAbruptCloseBeforeTheKeepAliveTimeoutIsDueIsAConnectionLoss() throws Exception {
+ startBrokerRecordingPublishes();
+ MqttEndpoint dropped = keepAliveEndpoint("dropped");
+ nanoTime.addAndGet(Duration.ofMillis(1_500).toNanos());
+
+ ArgumentCaptor<Handler<Void>> close = handlerCaptor();
+ verify(dropped).closeHandler(close.capture());
+ close.getValue().handle(null);
+
+ assertEquals("CONNECTION_LOST", awaitPresence("disconnected", "dropped", 1).get(0).get("disconnectReason").asText());
+ }
+
+ /**
+ * Vert.x times the keep-alive from the CONNECT, so a client its custom authorizer took a while
+ * to admit, silent since, has timed out when Vert.x closes it.
+ */
+ @Test
+ void aSilentClientTimesOutFromItsConnectNotFromItsCustomAuthorizersDecision() throws Exception {
+ startBrokerRecordingPublishes();
+ when(customAuthorizer.connectPolicies(eq("slow-auth"), eq("slow"), any(), any(), any())).thenAnswer(invocation -> {
+ nanoTime.addAndGet(Duration.ofMillis(1_500).toNanos());
+ return List.of("""
+ {"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Action": "iot:Connect", "Resource": "*"}]}""");
+ });
+ SocketAddress bridge = SocketAddress.inetSocketAddress(50125, "127.0.0.1");
+ broker.webSocketUpgrades().put(bridge, new IotCustomAuthorizer.WebSocketUpgrade(
+ Map.of(), IotCustomAuthorizer.NAME_PARAM + "=slow-auth", null, "203.0.113.7", 41235));
+ MqttEndpoint endpoint = mock(MqttEndpoint.class);
+ when(endpoint.clientIdentifier()).thenReturn("slow");
+ when(endpoint.remoteAddress()).thenReturn(bridge);
+ when(endpoint.keepAliveTimeSeconds()).thenReturn(1);
+
+ broker.handleEndpoint(endpoint, false);
+ awaitPresence("connected", "slow", 1);
+ nanoTime.addAndGet(Duration.ofMillis(600).toNanos());
+ ArgumentCaptor<Handler<Void>> close = handlerCaptor();
+ verify(endpoint, times(2)).closeHandler(close.capture());
+ close.getValue().handle(null);
+
+ assertEquals("MQTT_KEEP_ALIVE_TIMEOUT", awaitPresence("disconnected", "slow", 1).get(0).get("disconnectReason").asText());
+ }
+
+ /**
+ * A SUBSCRIBE or UNSUBSCRIBE the replaced connection still had in flight when a newer one took
+ * its client id over changes nothing: the newer session neither inherits the one nor loses a
+ * subscription to the other.
+ */
+ @Test
+ void aTakenOverSessionNeitherAddsNorRemovesSubscriptions() throws Exception {
+ startBrokerRecordingPublishes();
+ MqttEndpoint replaced = mock(MqttEndpoint.class);
+ when(replaced.clientIdentifier()).thenReturn("taken");
+ when(replaced.isCleanSession()).thenReturn(true);
+ broker.handleEndpoint(replaced, false);
+ ArgumentCaptor<Handler<MqttSubscribeMessage>> lateSubscribe = handlerCaptor();
+ verify(replaced).subscribeHandler(lateSubscribe.capture());
+ ArgumentCaptor<Handler<MqttUnsubscribeMessage>> lateUnsubscribe = handlerCaptor();
+ verify(replaced).unsubscribeHandler(lateUnsubscribe.capture());
+ MqttEndpoint current = mock(MqttEndpoint.class);
+ when(current.clientIdentifier()).thenReturn("taken");
+ when(current.isCleanSession()).thenReturn(true);
+ when(current.isConnected()).thenReturn(true);
+ broker.handleEndpoint(current, false);
+ ArgumentCaptor<Handler<MqttSubscribeMessage>> subscribe = handlerCaptor();
+ verify(current).subscribeHandler(subscribe.capture());
+ subscribe.getValue().handle(subscribeMessage("kept/topic"));
+
+ lateSubscribe.getValue().handle(subscribeMessage("stale/topic"));
+ MqttUnsubscribeMessage unsubscription = mock(MqttUnsubscribeMessage.class);
+ when(unsubscription.topics()).thenReturn(List.of("kept/topic"));
+ lateUnsubscribe.getValue().handle(unsubscription);
+ broker.publish("stale/topic", "{}".getBytes(StandardCharsets.UTF_8));
+ broker.publish("kept/topic", "{}".getBytes(StandardCharsets.UTF_8));
+
+ verify(current, timeout(10_000)).publish(eq("kept/topic"), any(), any(), anyBoolean(), anyBoolean());
+ verify(current, never()).publish(eq("stale/topic"), any(), any(), anyBoolean(), anyBoolean());
+ }
+
/**
* Two publishes from two connections, the first one's rule action held on a latch: the second
* publish is fanned out, so its rules were handed over, yet they start only once the first
@@ -688,6 +1365,42 @@ class IotMqttBrokerServiceTest {
assertTrue(waitingRan.get(), "the evaluation waiting at the lone afterReset() still ran");
}
+ /**
+ * A lifecycle event is published once the callback that ended its session returns. Here that
+ * callback runs a whole reset first, so the event, queued behind it on the same context, is
+ * still recorded but skips the rules it was admitted to before the reset.
+ */
+ @Test
+ void aLifecycleEventQueuedBeforeAResetSkipsItsRules() throws Exception {
+ String topic = "$aws/events/presence/disconnected/reset-race";
+ IotService service = mock(IotService.class);
+ when(iotService.get()).thenReturn(service);
+ AtomicBoolean rulesRan = new AtomicBoolean();
+ doAnswer(invocation -> {
+ invocation.<Executor>getArgument(6).execute(() -> rulesRan.set(true));
+ return null;
+ }).when(service).publish(eq(topic), any(), anyBoolean(), anyInt(), any(), any(), any());
+ MqttEndpoint endpoint = mock(MqttEndpoint.class);
+ when(endpoint.clientIdentifier()).thenReturn("reset-race");
+ broker.handleEndpoint(endpoint, false);
+ Context context = vertx.getOrCreateContext();
+ AtomicBoolean disconnected = new AtomicBoolean();
+ CountDownLatch eventHandled = new CountDownLatch(1);
+
+ context.runOnContext(ignored -> {
+ disconnected.set(broker.disconnectClient("reset-race", false));
+ broker.beforeReset();
+ broker.afterReset();
+ context.runOnContext(queued -> eventHandled.countDown());
+ });
+
+ assertTrue(eventHandled.await(10, TimeUnit.SECONDS), "the queued lifecycle event was handled");
+ broker.ruleWorker.executeBlocking(() -> null, false).toCompletionStage().toCompletableFuture().get(10, TimeUnit.SECONDS);
+ assertTrue(disconnected.get(), "DeleteConnection ended the session");
+ verify(service).publish(eq(topic), any(), anyBoolean(), anyInt(), any(), any(), any());
+ assertFalse(rulesRan.get(), "the event's rules, admitted before the reset, were skipped");
+ }
+
/**
* A restart whose TLS listener cannot bind fails and leaves rules off; a later successful start
* turns them on again.
@@ -770,7 +1483,7 @@ class IotMqttBrokerServiceTest {
when(serviceConfig.services().iot().ruleSqlStrict()).thenReturn(false);
ObjectMapper mapper = new ObjectMapper();
IotService service = new IotService(storageFactory, serviceConfig, new RegionResolver("eu-west-1", "111122223333"),
- mapper, new IotPublishEventRecorder(), broker, mock(SqsService.class), mock(SnsService.class),
+ mapper, recorder, broker, mock(SqsService.class), mock(SnsService.class),
mock(S3Service.class), mock(KinesisService.class),
new DynamoDbFacade(items, mock(DynamoDbTableAccess.class), new RegionResolver("us-east-1", "000000000000")),
mock(LambdaService.class), mock(FirehoseService.class), mock(CloudWatchLogsService.class), ca,
@@ -783,6 +1496,182 @@ class IotMqttBrokerServiceTest {
broker.startIfEnabled();
}
+ /** Starts the broker over a real {@link IotService} whose recorder sees every publish the broker hands it. */
+ private void startBrokerRecordingPublishes() throws Exception {
+ startBrokerWithRuleWriting(mock(DynamoDbItemAccess.class));
+ }
+
+ /** The recorded presence events of one type for one client id, in publish order. */
+ private List<JsonNode> presence(String eventType, String clientId) {
+ String topic = "$aws/events/presence/" + eventType + "/" + clientId;
+ return recorder.recentEvents().stream()
+ .filter(event -> event.topic().equals(topic))
+ .map(event -> json(event.payload()))
+ .toList();
+ }
+
+ /** Every recorded presence event for one client id, in publish order. */
+ private List<JsonNode> presenceOf(String clientId) {
+ return recorder.recentEvents().stream()
+ .filter(event -> event.topic().equals("$aws/events/presence/connected/" + clientId)
+ || event.topic().equals("$aws/events/presence/disconnected/" + clientId))
+ .map(event -> json(event.payload()))
+ .toList();
+ }
+
+ private List<JsonNode> awaitPresence(String eventType, String clientId, int count) throws InterruptedException {
+ long deadline = System.currentTimeMillis() + 10_000;
+ List<JsonNode> events = presence(eventType, clientId);
+ while (events.size() < count) {
+ if (System.currentTimeMillis() > deadline) {
+ throw new AssertionError("expected " + count + " " + eventType + " events for " + clientId + ", got " + events);
+ }
+ Thread.sleep(10);
+ events = presence(eventType, clientId);
+ }
+ return events;
+ }
+
+ /**
+ * Connects and cleanly disconnects one more client and waits for its disconnected event. The
+ * plaintext listener handles every connection on one event loop, so by then it has also handled
+ * whatever the earlier connections sent and their closes.
+ */
+ private void barrier() throws Exception {
+ String clientId = "barrier-" + System.nanoTime();
+ disconnect(connectPlain(clientId));
+ awaitPresence("disconnected", clientId, 1);
+ }
+
+ private static void awaitDisconnected(MqttClient client) throws InterruptedException {
+ long deadline = System.currentTimeMillis() + 10_000;
+ while (client.isConnected()) {
+ if (System.currentTimeMillis() > deadline) {
+ throw new AssertionError("client " + client.getClientId() + " is still connected");
+ }
+ Thread.sleep(10);
+ }
+ }
+
+ private static MqttSubscribeMessage subscribeMessage(String topicFilter) {
+ MqttTopicSubscription requested = mock(MqttTopicSubscription.class);
+ when(requested.topicName()).thenReturn(topicFilter);
+ when(requested.qualityOfService()).thenReturn(MqttQoS.AT_MOST_ONCE);
+ MqttSubscribeMessage message = mock(MqttSubscribeMessage.class);
+ when(message.topicSubscriptions()).thenReturn(List.of(requested));
+ return message;
+ }
+
+ /** A connected client with a one-second keep-alive. */
+ private MqttEndpoint keepAliveEndpoint(String clientId) {
+ MqttEndpoint endpoint = mock(MqttEndpoint.class);
+ when(endpoint.clientIdentifier()).thenReturn(clientId);
+ when(endpoint.keepAliveTimeSeconds()).thenReturn(1);
+ broker.handleEndpoint(endpoint, false);
+ return endpoint;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <T> ArgumentCaptor<Handler<T>> handlerCaptor() {
+ return ArgumentCaptor.forClass(Handler.class);
+ }
+
+ private static List<String> eventTypes(List<JsonNode> events) {
+ return events.stream().map(event -> event.get("eventType").asText()).toList();
+ }
+
+ private static List<String> fieldNames(JsonNode node) {
+ List<String> names = new ArrayList<>();
+ node.fieldNames().forEachRemaining(names::add);
+ return names;
+ }
+
+ private static void assertDashedUuid(String value) {
+ assertEquals(UUID.fromString(value).toString(), value, "a dashed lowercase UUID");
+ }
+
+ private static JsonNode json(byte[] payload) {
+ try {
+ return MAPPER.readTree(payload);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ /** An MQTT 3.1.1 CONNECT over a plain socket, so the test decides exactly which packets the broker sees. */
+ private Socket rawConnect(String clientId) throws IOException {
+ Socket socket = new Socket();
+ socket.connect(new InetSocketAddress("127.0.0.1", plainPort), 5_000);
+ socket.setSoTimeout(5_000);
+ ByteArrayOutputStream body = new ByteArrayOutputStream();
+ writeUtf8(body, "MQTT");
+ body.write(0x04);
+ body.write(0x02);
+ body.write(0x00);
+ body.write(0x3c);
+ writeUtf8(body, clientId);
+ socket.getOutputStream().write(packet(0x10, body.toByteArray()));
+ assertArrayEquals(new byte[] {0x20, 0x02, 0x00, 0x00}, socket.getInputStream().readNBytes(4), "CONNACK accepted");
+ return socket;
+ }
+
+ private static void rawPublish(Socket socket, int qos, String topic, byte[] payload) throws IOException {
+ ByteArrayOutputStream body = new ByteArrayOutputStream();
+ writeUtf8(body, topic);
+ if (qos > 0) {
+ body.write(0x00);
+ body.write(0x01);
+ }
+ body.writeBytes(payload);
+ socket.getOutputStream().write(packet(0x30 | (qos << 1), body.toByteArray()));
+ }
+
+ private static byte[] packet(int type, byte[] body) {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ out.write(type);
+ int remaining = body.length;
+ do {
+ int encoded = remaining % 128;
+ remaining /= 128;
+ if (remaining > 0) {
+ encoded |= 128;
+ }
+ out.write(encoded);
+ } while (remaining > 0);
+ out.writeBytes(body);
+ return out.toByteArray();
+ }
+
+ private static void writeUtf8(ByteArrayOutputStream out, String value) {
+ byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+ out.write(bytes.length >>> 8);
+ out.write(bytes.length & 0xff);
+ out.writeBytes(bytes);
+ }
+
+ /** A client subscribed to one filter whose callback records the topic of every message the broker sends it. */
+ private MqttClient subscribedPlain(String clientId, String topicFilter, BlockingQueue<String> topics) throws MqttException {
+ MqttClient client = connectPlain(clientId);
+ client.setCallback(new MqttCallback() {
+ @Override
+ public void connectionLost(Throwable cause) {
+ // The test disconnects the client itself.
+ }
+
+ @Override
+ public void messageArrived(String topic, MqttMessage message) {
+ topics.add(topic);
+ }
+
+ @Override
+ public void deliveryComplete(IMqttDeliveryToken token) {
+ // Subscribers do not publish.
+ }
+ });
+ client.subscribe(topicFilter, 0);
+ return client;
+ }
+
private MqttClient connectPlain(String clientId) throws MqttException {
MqttClient client = new MqttClient("tcp://127.0.0.1:" + plainPort, clientId, new MemoryPersistence());
MqttConnectOptions options = new MqttConnectOptions();
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java
index 5cb1d1cba5..2327dfb456 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java
@@ -20,16 +20,21 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Random;
+import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
@@ -38,6 +43,7 @@ import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -50,6 +56,10 @@ class IotMqttEnabledIntegrationTest {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String BUILDING_RULE_SQL = "SELECT *, topic() AS topic, clientid() AS cid "
+ "FROM '$aws/things/+/shadow/name/building/update/accepted' WHERE endswith(clientToken, 'inbound')";
+ private static final String TKLOCAL_CONNECTED_SQL =
+ "SELECT * FROM '$aws/events/presence/connected/+' WHERE startswith(clientId, 'tklocal-')";
+ private static final String TKLOCAL_DISCONNECTED_SQL =
+ "SELECT * FROM '$aws/events/presence/disconnected/+' WHERE startswith(clientId, 'tklocal-')";
@Inject
IotPublishEventRecorder eventRecorder;
@@ -581,6 +591,46 @@ class IotMqttEnabledIntegrationTest {
}
}
+ @Test
+ void tklocalPresenceRulesReceiveTheConnectAndCleanDisconnectOfATklocalClientOnly() throws Exception {
+ long suffix = System.nanoTime();
+ String queueUrl = createQueue("presence-clean-" + suffix);
+ String connectedRule = "presenceConnected" + suffix;
+ String disconnectedRule = "presenceDisconnected" + suffix;
+ createSqsRule(connectedRule, TKLOCAL_CONNECTED_SQL, queueUrl);
+ createSqsRule(disconnectedRule, TKLOCAL_DISCONNECTED_SQL, queueUrl);
+ try {
+ String clientId = "tklocal-" + suffix;
+ try (MqttTestClient other = connectMqttClientId("other-" + suffix)) {
+ assertTrue(other.isConnected());
+ }
+ try (MqttTestClient client = connectMqttClientId(clientId)) {
+ assertTrue(client.isConnected());
+ }
+
+ // One worker runs the rules in publish order, so the other client's events were evaluated first.
+ List<JsonNode> messages = awaitMessages(queueUrl, 2);
+ assertEquals(2, messages.size(), "only the tklocal client's two events: " + messages);
+ Map<String, JsonNode> byType = new HashMap<>();
+ for (JsonNode message : messages) {
+ assertEquals(clientId, message.path("clientId").asText(), message.toString());
+ byType.put(message.path("eventType").asText(), message);
+ }
+ JsonNode connected = byType.get("connected");
+ JsonNode disconnected = byType.get("disconnected");
+ assertEquals(Set.of("connected", "disconnected"), byType.keySet());
+ assertEquals(connected.path("sessionIdentifier").asText(), disconnected.path("sessionIdentifier").asText());
+ assertEquals(0, connected.path("versionNumber").asLong());
+ assertEquals(0, disconnected.path("versionNumber").asLong());
+ assertEquals("127.0.0.1", connected.path("ipAddress").asText());
+ assertTrue(disconnected.path("clientInitiatedDisconnect").asBoolean(), disconnected.toString());
+ assertEquals("CLIENT_INITIATED_DISCONNECT", disconnected.path("disconnectReason").asText());
+ } finally {
+ deleteRule(connectedRule);
+ deleteRule(disconnectedRule);
+ }
+ }
+
@Test
void mqttSubscriberReceivesTheAcceptedEventOfARestShadowUpdate() throws Exception {
String thing = "shadowRestToMqtt" + System.nanoTime();
@@ -600,6 +650,74 @@ class IotMqttEnabledIntegrationTest {
}
}
+ @Test
+ void tklocalDisconnectedRuleReceivesConnectionLostWhenTheClientVanishesWithoutDisconnect() throws Exception {
+ long suffix = System.nanoTime();
+ String queueUrl = createQueue("presence-lost-" + suffix);
+ String disconnectedRule = "presenceLost" + suffix;
+ createSqsRule(disconnectedRule, TKLOCAL_DISCONNECTED_SQL, queueUrl);
+ try {
+ String clientId = "tklocal-lost-" + suffix;
+ MqttTestClient client = connectMqttClientId(clientId);
+ client.closeWithoutDisconnect();
+
+ List<JsonNode> messages = awaitMessages(queueUrl, 1);
+ assertEquals(1, messages.size(), messages.toString());
+ JsonNode disconnected = messages.get(0);
+ assertEquals(clientId, disconnected.path("clientId").asText());
+ assertEquals("CONNECTION_LOST", disconnected.path("disconnectReason").asText());
+ assertTrue(disconnected.path("clientInitiatedDisconnect").isBoolean(), disconnected.toString());
+ assertFalse(disconnected.path("clientInitiatedDisconnect").asBoolean());
+ } finally {
+ deleteRule(disconnectedRule);
+ }
+ }
+
+ @Test
+ void tklocalDisconnectedRuleReceivesKeepAliveTimeoutWhenTheClientFallsSilent() throws Exception {
+ long suffix = System.nanoTime();
+ String queueUrl = createQueue("presence-silent-" + suffix);
+ String disconnectedRule = "presenceSilent" + suffix;
+ createSqsRule(disconnectedRule, TKLOCAL_DISCONNECTED_SQL, queueUrl);
+ try (Socket socket = rawConnect("tklocal-silent-" + suffix, 1)) {
+ List<JsonNode> messages = awaitMessages(queueUrl, 1);
+
+ assertEquals(1, messages.size(), messages.toString());
+ JsonNode disconnected = messages.get(0);
+ assertEquals("tklocal-silent-" + suffix, disconnected.path("clientId").asText());
+ assertEquals("MQTT_KEEP_ALIVE_TIMEOUT", disconnected.path("disconnectReason").asText());
+ assertTrue(disconnected.path("clientInitiatedDisconnect").isBoolean(), disconnected.toString());
+ assertFalse(disconnected.path("clientInitiatedDisconnect").asBoolean());
+ } finally {
+ deleteRule(disconnectedRule);
+ }
+ }
+
+ @Test
+ void tklocalDisconnectedRuleReceivesConnectionLostWhenAPingingClientDrops() throws Exception {
+ long suffix = System.nanoTime();
+ String queueUrl = createQueue("presence-pinging-" + suffix);
+ String disconnectedRule = "presencePinging" + suffix;
+ createSqsRule(disconnectedRule, TKLOCAL_DISCONNECTED_SQL, queueUrl);
+ try {
+ try (Socket socket = rawConnect("tklocal-pinging-" + suffix, 1)) {
+ Instant pingUntil = Instant.now().plusMillis(2_500);
+ while (Instant.now().isBefore(pingUntil)) {
+ socket.getOutputStream().write(new byte[] {(byte) 0xc0, 0x00});
+ assertArrayEquals(new byte[] {(byte) 0xd0, 0x00}, socket.getInputStream().readNBytes(2), "PINGRESP");
+ Thread.sleep(400);
+ }
+ }
+
+ List<JsonNode> messages = awaitMessages(queueUrl, 1);
+ assertEquals(1, messages.size(), messages.toString());
+ assertEquals("CONNECTION_LOST", messages.get(0).path("disconnectReason").asText());
+ assertFalse(messages.get(0).path("clientInitiatedDisconnect").asBoolean());
+ } finally {
+ deleteRule(disconnectedRule);
+ }
+ }
+
private String createQueue(String queueName) {
return given()
.contentType("application/x-www-form-urlencoded")
@@ -638,12 +756,31 @@ class IotMqttEnabledIntegrationTest {
throw new AssertionError("No message arrived on " + queueUrl);
}
+ /** Receives until {@code count} messages arrived or ten seconds passed, then once more, so a surplus one shows too. */
+ private List<JsonNode> awaitMessages(String queueUrl, int count) throws Exception {
+ List<JsonNode> messages = new ArrayList<>();
+ Instant deadline = Instant.now().plus(Duration.ofSeconds(10));
+ while (messages.size() < count && Instant.now().isBefore(deadline)) {
+ for (String body : receiveMessages(queueUrl)) {
+ messages.add(OBJECT_MAPPER.readTree(body));
+ }
+ if (messages.size() < count) {
+ Thread.sleep(50);
+ }
+ }
+ for (String body : receiveMessages(queueUrl)) {
+ messages.add(OBJECT_MAPPER.readTree(body));
+ }
+ return messages;
+ }
+
private void createSqsRule(String ruleName, String sql, String queueUrl) {
ObjectNode sqs = OBJECT_MAPPER.createObjectNode()
.put("roleArn", "arn:aws:iam::000000000000:role/iot-rule-role")
.put("queueUrl", queueUrl);
ObjectNode payload = OBJECT_MAPPER.createObjectNode()
.put("sql", sql)
+ .put("awsIotSqlVersion", "2016-03-23")
.put("ruleDisabled", false);
payload.putArray("actions").addObject().set("sqs", sqs);
ObjectNode body = OBJECT_MAPPER.createObjectNode();
@@ -726,6 +863,34 @@ class IotMqttEnabledIntegrationTest {
throw new AssertionError("MQTT publish event was not recorded");
}
+ /** An MQTT 3.1.1 CONNECT over a plain socket, so the test decides exactly which packets follow it. */
+ private static Socket rawConnect(String clientId, int keepAliveSeconds) throws IOException {
+ Socket socket = new Socket();
+ socket.connect(new InetSocketAddress("127.0.0.1", IotMqttTlsProfile.PLAIN_PORT), 5_000);
+ socket.setSoTimeout(5_000);
+ ByteArrayOutputStream body = new ByteArrayOutputStream();
+ writeUtf8(body, "MQTT");
+ body.write(0x04);
+ body.write(0x02);
+ body.write(keepAliveSeconds >>> 8);
+ body.write(keepAliveSeconds & 0xff);
+ writeUtf8(body, clientId);
+ ByteArrayOutputStream connect = new ByteArrayOutputStream();
+ connect.write(0x10);
+ connect.write(body.size());
+ connect.writeBytes(body.toByteArray());
+ socket.getOutputStream().write(connect.toByteArray());
+ assertArrayEquals(new byte[] {0x20, 0x02, 0x00, 0x00}, socket.getInputStream().readNBytes(4), "CONNACK accepted");
+ return socket;
+ }
+
+ private static void writeUtf8(ByteArrayOutputStream out, String value) {
+ byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+ out.write(bytes.length >>> 8);
+ out.write(bytes.length & 0xff);
+ out.writeBytes(bytes);
+ }
+
private record MqttPublish(String topic, byte[] payload) {
}
@@ -795,6 +960,12 @@ class IotMqttEnabledIntegrationTest {
throw new AssertionError("MQTT connection stayed open");
}
+ /** Drops the TCP connection without sending DISCONNECT, as a crashed device would. */
+ void closeWithoutDisconnect() throws MqttException {
+ client.disconnectForcibly(0, 0, false);
+ client.close();
+ }
+
@Override
public void close() throws MqttException {
if (client.isConnected()) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttWebSocketBridgeTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttWebSocketBridgeTest.java
index 037943dfe7..605083cf70 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttWebSocketBridgeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttWebSocketBridgeTest.java
@@ -142,8 +142,8 @@ class IotMqttWebSocketBridgeTest {
@Test
void closingAnOlderSessionKeepsANewerSessionsEqualUpgradeOnTheSameAddress() {
SocketAddress address = SocketAddress.inetSocketAddress(50000, "127.0.0.1");
- IotCustomAuthorizer.WebSocketUpgrade older = new IotCustomAuthorizer.WebSocketUpgrade(Map.of("host", "h"), "", null, null);
- IotCustomAuthorizer.WebSocketUpgrade newer = new IotCustomAuthorizer.WebSocketUpgrade(Map.of("host", "h"), "", null, null);
+ IotCustomAuthorizer.WebSocketUpgrade older = new IotCustomAuthorizer.WebSocketUpgrade(Map.of("host", "h"), "", null, null, -1);
+ IotCustomAuthorizer.WebSocketUpgrade newer = new IotCustomAuthorizer.WebSocketUpgrade(Map.of("host", "h"), "", null, null, -1);
Map<SocketAddress, IotCustomAuthorizer.WebSocketUpgrade> upgrades = new ConcurrentHashMap<>(Map.of(address, newer));
IotMqttWebSocketBridge.forgetting(upgrades, address, older).run();
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java
index 604e403de1..a4b684b522 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java
@@ -206,6 +206,28 @@ class IotServiceTest {
verify(lambda).invoke(eq(REGION), eq(FUNCTION_ARN), any(), eq(InvocationType.Event));
}
+ /** As AWS IoT does for rule topic filters: a wildcard first level does not match a lifecycle topic. */
+ @Test
+ void aRuleWhoseTopicFilterStartsWithAWildcardDoesNotFireForALifecycleTopic() throws Exception {
+ String everything = "http://localhost:4566/000000000000/everything";
+ String anyPresence = "http://localhost:4566/000000000000/any-presence";
+ String presence = "http://localhost:4566/000000000000/presence";
+ createRule("everything", sqsRule("SELECT * FROM '#'", everything));
+ createRule("anyPresence", sqsRule("SELECT * FROM '+/events/presence/+/+'", anyPresence));
+ createRule("presence", sqsRule("SELECT * FROM '$aws/events/presence/connected/+'", presence));
+
+ service.handlePublish("$aws/events/presence/connected/x", "{\"clientId\":\"x\"}".getBytes(StandardCharsets.UTF_8),
+ true, REGION, "x", Runnable::run);
+
+ verify(sqs, never()).sendMessage(eq(everything), anyString(), anyInt(), anyString());
+ verify(sqs, never()).sendMessage(eq(anyPresence), anyString(), anyInt(), anyString());
+ verify(sqs).sendMessage(eq(presence), eq("{\"clientId\":\"x\"}"), eq(0), eq(REGION));
+
+ service.handlePublish("a/b", "{\"v\":1}".getBytes(StandardCharsets.UTF_8), true, REGION, null, Runnable::run);
+
+ verify(sqs).sendMessage(eq(everything), eq("{\"v\":1}"), eq(0), eq(REGION));
+ }
+
@Test
void aFailingActionDoesNotFailThePublishOrStopTheOtherActions() throws Exception {
createRule("metricsRule", sqsThenLambdaRule(null));