Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(iot): accept MQTT packets up to the AWS 128 KB payload limit (#4307)

Mario Toffiaec40bcf5f47e
diff --git a/docs/services/iot.md b/docs/services/iot.md
index be3a5a3341..1a17698f91 100644
--- a/docs/services/iot.md
+++ b/docs/services/iot.md
@@ -78,6 +78,7 @@ Broker scope:
- Target real AWS IoT/device SDK style MQTT clients, not only handcrafted packet tests.
- Support MQTT v3 and MQTT 5 CONNECT handling used by local compatibility tests.
- Support QoS 0 and QoS 1 publish/subscribe behavior for the local AWS IoT slice.
+- Accept PUBLISH payloads up to 128 KB on 1883, 8883 and `/mqtt`, the AWS IoT Core quota (packets up to 146 KB including the variable header). A larger publish disconnects the client without acknowledgement, delivery or rule evaluation, as on AWS.
- Serve MQTT over TLS on 8883 next to plaintext 1883 when TLS is enabled, and verify the device certificate and its `iot:Connect` permission there.
- Keep the plaintext listener permissive; topic-level authorization (`Publish`, `Subscribe`, `Receive`) is follow-up scope.
- Keep MQTT broker logging minimal.
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
index 598b9fcbbe..7f0022da8a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
@@ -94,6 +94,12 @@ public class IotMqttBrokerService {
}
};
+ /** AWS IoT Core's MQTT payload quota; a larger PUBLISH disconnects the client unacknowledged. */
+ private static final int MAX_PAYLOAD_SIZE = 128 * 1024;
+
+ /** AWS IoT Core's MQTT packet quota, counted as Netty's decoder counts: variable header plus payload. */
+ private static final int MAX_PACKET_SIZE = 146 * 1024;
+
private static final Pattern IPV4_LITERAL = Pattern.compile("\\d{1,3}(\\.\\d{1,3}){3}");
private final EmulatorConfig config;
@@ -141,7 +147,8 @@ public class IotMqttBrokerService {
MqttServer mqttServer = listen(new MqttServerOptions()
.setHost(config.services().iot().mqtt().host())
- .setPort(config.services().iot().mqtt().port()), "IoT MQTT broker", false);
+ .setPort(config.services().iot().mqtt().port())
+ .setMaxMessageSize(MAX_PACKET_SIZE), "IoT MQTT broker", false);
try {
startTlsListener();
} catch (RuntimeException e) {
@@ -176,6 +183,7 @@ public class IotMqttBrokerService {
tlsServer = listen(new MqttServerOptions()
.setHost(config.services().iot().mqtt().host())
.setPort(tlsPort)
+ .setMaxMessageSize(MAX_PACKET_SIZE)
.setSsl(true)
.setKeyCertOptions(KeyCertOptions.wrap(manager))
.setTrustOptions(TrustOptions.wrap(ACCEPT_ANY_CLIENT_CERTIFICATE))
@@ -437,6 +445,12 @@ public class IotMqttBrokerService {
}
private void handlePublish(ClientSession session, MqttPublishMessage message) {
+ if (message.payload().length() > MAX_PAYLOAD_SIZE) {
+ LOG.debugv("IoT MQTT client {0} disconnected: {1} byte payload on {2} exceeds the AWS limit",
+ session.clientId(), Integer.toString(message.payload().length()), message.topicName());
+ session.endpoint().close();
+ return;
+ }
byte[] payload = message.payload().getBytes();
if (message.qosLevel() == MqttQoS.EXACTLY_ONCE) {
session.endpoint().close();
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java
index 0ea12acd48..5c4dd66579 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java
@@ -23,6 +23,7 @@ import java.time.Instant;
import java.util.Arrays;
import java.util.Map;
import java.util.Optional;
+import java.util.Random;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
@@ -31,12 +32,14 @@ import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@QuarkusTest
@TestProfile(IotMqttEnabledIntegrationTest.EnabledMqttProfile.class)
class IotMqttEnabledIntegrationTest {
+ static final int AWS_MAX_PAYLOAD = 128 * 1024;
private static final int PORT = 18831;
private static final String BROKER_URI = "tcp://127.0.0.1:" + PORT;
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@@ -176,6 +179,79 @@ class IotMqttEnabledIntegrationTest {
awaitPublishedEvent(topic, payload);
}
+ @Test
+ void publishOfTheAwsMaximumPayloadIsDelivered() throws Exception {
+ String topic = "phase6/limit/" + System.nanoTime();
+ byte[] payload = randomPayload(AWS_MAX_PAYLOAD);
+
+ try (MqttTestClient subscriber = connectMqtt("limit-sub")) {
+ subscriber.subscribe(topic, 1);
+ try (MqttTestClient publisher = connectMqtt("limit-pub")) {
+ publisher.publish(topic, payload, 1);
+ }
+ assertArrayEquals(payload, subscriber.takePublish().payload());
+ }
+
+ awaitPublishedEvent(topic, payload);
+ }
+
+ @Test
+ void publishAboveTheAwsMaximumPayloadDisconnectsWithoutAckOrDelivery() throws Exception {
+ String topic = "phase6/over-limit/" + System.nanoTime();
+ byte[] after = "after".getBytes(StandardCharsets.UTF_8);
+
+ try (MqttTestClient subscriber = connectMqtt("over-limit-sub")) {
+ subscriber.subscribe(topic, 1);
+ try (MqttTestClient publisher = connectMqtt("over-limit-pub")) {
+ assertThrows(MqttException.class, () -> publisher.publish(topic, randomPayload(AWS_MAX_PAYLOAD + 1), 1));
+ publisher.awaitDisconnected();
+ }
+ try (MqttTestClient next = connectMqtt("over-limit-next")) {
+ next.publish(topic, after);
+ }
+ assertArrayEquals(after, subscriber.takePublish().payload(), "the oversized publish is never delivered");
+ }
+
+ awaitPublishedEvent(topic, after);
+ assertEquals(1, eventRecorder.recentEvents().stream().filter(event -> topic.equals(event.topic())).count(),
+ "the oversized publish reaches no rule");
+ }
+
+ @Test
+ void mqtt5PublishAtTheAwsPropertyAndPayloadMaximumsIsDelivered() throws Exception {
+ String topic = "phase6/mqtt5-limit/" + System.nanoTime();
+ byte[] payload = randomPayload(AWS_MAX_PAYLOAD);
+
+ try (Mqtt5TestClient subscriber = connectMqtt5("mqtt5-limit-sub")) {
+ subscriber.subscribe(topic);
+ try (Mqtt5TestClient publisher = connectMqtt5("mqtt5-limit-pub")) {
+ publisher.publish(topic, payload, 1, awsMaximumProperties(8 * 1024));
+ }
+ assertArrayEquals(payload, subscriber.takePublish().payload());
+ }
+ }
+
+ @Test
+ void mqtt5PublishAboveTheAwsPacketMaximumDisconnectsWithoutAck() throws Exception {
+ String topic = "phase6/mqtt5-over-limit/" + System.nanoTime();
+
+ try (Mqtt5TestClient publisher = connectMqtt5("mqtt5-over-limit-pub")) {
+ org.eclipse.paho.mqttv5.common.MqttException thrown = assertThrows(
+ org.eclipse.paho.mqttv5.common.MqttException.class,
+ () -> publisher.publish(topic, randomPayload(AWS_MAX_PAYLOAD), 1, awsMaximumProperties(16 * 1024)));
+ assertEquals(org.eclipse.paho.mqttv5.client.MqttClientException.REASON_CODE_CONNECTION_LOST, thrown.getReasonCode());
+ }
+ }
+
+ /** 8 KB correlation data plus user properties of the given key and value size, as AWS counts them. */
+ private static org.eclipse.paho.mqttv5.common.packet.MqttProperties awsMaximumProperties(int userPropertiesSize) {
+ org.eclipse.paho.mqttv5.common.packet.MqttProperties properties = new org.eclipse.paho.mqttv5.common.packet.MqttProperties();
+ properties.setCorrelationData(randomPayload(8 * 1024));
+ properties.setUserProperties(java.util.List.of(
+ new org.eclipse.paho.mqttv5.common.packet.UserProperty("k", "v".repeat(userPropertiesSize - 1))));
+ return properties;
+ }
+
@Test
void shadowUpdateTopicPublishesAcceptedResponse() throws Exception {
try (MqttTestClient subscriber = connectMqtt("phase7-update-sub")) {
@@ -408,6 +484,12 @@ class IotMqttEnabledIntegrationTest {
return Mqtt5TestClient.connect(uniqueClientId(clientId));
}
+ static byte[] randomPayload(int size) {
+ byte[] payload = new byte[size];
+ new Random(size).nextBytes(payload);
+ return payload;
+ }
+
private byte[] json(String value) {
return value.getBytes(StandardCharsets.UTF_8);
}
@@ -442,6 +524,7 @@ class IotMqttEnabledIntegrationTest {
static MqttTestClient connect(String clientId) throws MqttException {
MqttClient client = new MqttClient(BROKER_URI, clientId, new MemoryPersistence());
+ client.setTimeToWait(10_000);
MqttTestClient testClient = new MqttTestClient(client);
client.setCallback(new MqttCallback() {
@Override
@@ -494,7 +577,7 @@ class IotMqttEnabledIntegrationTest {
}
Thread.sleep(25);
}
- throw new AssertionError("MQTT connection stayed open after DeleteConnection");
+ throw new AssertionError("MQTT connection stayed open");
}
@Override
@@ -553,6 +636,7 @@ class IotMqttEnabledIntegrationTest {
public void authPacketArrived(int reasonCode, org.eclipse.paho.mqttv5.common.packet.MqttProperties properties) {
}
});
+ client.setTimeToWait(10_000);
client.connect(connectOptions());
return testClient;
}
@@ -566,8 +650,14 @@ class IotMqttEnabledIntegrationTest {
}
void publish(String topic, byte[] payload) throws org.eclipse.paho.mqttv5.common.MqttException {
+ publish(topic, payload, 0, new org.eclipse.paho.mqttv5.common.packet.MqttProperties());
+ }
+
+ void publish(String topic, byte[] payload, int qos, org.eclipse.paho.mqttv5.common.packet.MqttProperties properties)
+ throws org.eclipse.paho.mqttv5.common.MqttException {
org.eclipse.paho.mqttv5.common.MqttMessage message = new org.eclipse.paho.mqttv5.common.MqttMessage(payload);
- message.setQos(0);
+ message.setQos(qos);
+ message.setProperties(properties);
client.publish(topic, message);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttTlsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttTlsIntegrationTest.java
index b453f7f813..9965ea187b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttTlsIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttTlsIntegrationTest.java
@@ -38,6 +38,8 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import static io.github.hectorvent.floci.services.iot.IotMqttEnabledIntegrationTest.AWS_MAX_PAYLOAD;
+import static io.github.hectorvent.floci.services.iot.IotMqttEnabledIntegrationTest.randomPayload;
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
@@ -213,6 +215,38 @@ class IotMqttTlsIntegrationTest {
}
}
+ @Test
+ void publishOfTheAwsMaximumPayloadIsDeliveredOverTls() throws Exception {
+ String topic = "tls/limit/" + System.nanoTime();
+ byte[] payload = randomPayload(AWS_MAX_PAYLOAD);
+
+ try (TlsClient subscriber = TlsClient.connect("tls-limit-sub-" + System.nanoTime())) {
+ subscriber.subscribe(topic, 1);
+ try (TlsClient publisher = TlsClient.connect("tls-limit-pub-" + System.nanoTime())) {
+ publisher.publish(topic, payload, 1);
+ }
+ assertArrayEquals(payload, subscriber.takePayload());
+ }
+ }
+
+ @Test
+ void publishAboveTheAwsMaximumPayloadDisconnectsOverTls() throws Exception {
+ String topic = "tls/over-limit/" + System.nanoTime();
+ byte[] after = "after".getBytes(StandardCharsets.UTF_8);
+
+ try (TlsClient subscriber = TlsClient.connect("tls-over-sub-" + System.nanoTime())) {
+ subscriber.subscribe(topic, 1);
+ try (TlsClient publisher = TlsClient.connect("tls-over-pub-" + System.nanoTime())) {
+ assertThrows(MqttException.class, () -> publisher.publish(topic, randomPayload(AWS_MAX_PAYLOAD + 1), 1));
+ publisher.awaitDisconnected();
+ }
+ try (TlsClient next = TlsClient.connect("tls-over-next-" + System.nanoTime())) {
+ next.publish(topic, after, 0);
+ }
+ assertArrayEquals(after, subscriber.takePayload(), "the oversized publish is never delivered");
+ }
+ }
+
@Test
void aRestartOfTheBrokerServesTlsAgain() throws Exception {
broker.stop();
@@ -263,6 +297,7 @@ class IotMqttTlsIntegrationTest {
static TlsClient connect(String clientId) throws Exception {
MqttClient client = new MqttClient("ssl://127.0.0.1:" + TLS_PORT, clientId, new MemoryPersistence());
+ client.setTimeToWait(10_000);
TlsClient tlsClient = new TlsClient(client);
client.setCallback(new MqttCallback() {
@Override
@@ -313,7 +348,7 @@ class IotMqttTlsIntegrationTest {
}
Thread.sleep(25);
}
- throw new AssertionError("the TLS session stayed open after its client id reconnected elsewhere");
+ throw new AssertionError("the TLS session stayed open");
}
@Override
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttWebSocketIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttWebSocketIntegrationTest.java
index 300b92569b..94fd2303c0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttWebSocketIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttWebSocketIntegrationTest.java
@@ -46,6 +46,8 @@ import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import static io.github.hectorvent.floci.services.iot.IotMqttEnabledIntegrationTest.AWS_MAX_PAYLOAD;
+import static io.github.hectorvent.floci.services.iot.IotMqttEnabledIntegrationTest.randomPayload;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
@@ -231,6 +233,38 @@ public class IotMqttWebSocketIntegrationTest {
}
}
+ @Test
+ void publishOfTheAwsMaximumPayloadIsDeliveredOverWss() throws Exception {
+ String topic = "ws/limit/" + System.nanoTime();
+ byte[] payload = randomPayload(AWS_MAX_PAYLOAD);
+
+ try (WsClient subscriber = WsClient.connect(wss("/mqtt"), "ws-limit-sub-" + System.nanoTime(), null, null)) {
+ subscriber.subscribe(topic);
+ try (WsClient publisher = WsClient.connect(wss("/mqtt"), "ws-limit-pub-" + System.nanoTime(), null, null)) {
+ publisher.publish(topic, payload, 1);
+ }
+ assertArrayEquals(payload, subscriber.takePayload());
+ }
+ }
+
+ @Test
+ void publishAboveTheAwsMaximumPayloadDisconnectsOverWss() throws Exception {
+ String topic = "ws/over-limit/" + System.nanoTime();
+ byte[] after = "after".getBytes(StandardCharsets.UTF_8);
+
+ try (WsClient subscriber = WsClient.connect(wss("/mqtt"), "ws-over-sub-" + System.nanoTime(), null, null)) {
+ subscriber.subscribe(topic);
+ try (WsClient publisher = WsClient.connect(wss("/mqtt"), "ws-over-pub-" + System.nanoTime(), null, null)) {
+ assertThrows(MqttException.class, () -> publisher.publish(topic, randomPayload(AWS_MAX_PAYLOAD + 1), 1));
+ assertTrue(publisher.awaitConnectionLost(10, TimeUnit.SECONDS));
+ }
+ try (WsClient next = WsClient.connect(wss("/mqtt"), "ws-over-next-" + System.nanoTime(), null, null)) {
+ next.publish(topic, after, 0);
+ }
+ assertArrayEquals(after, subscriber.takePayload(), "the oversized publish is never delivered");
+ }
+ }
+
@Test
void manyConcurrentWebSocketClientsEachReceiveThePublish() throws Exception {
String topic = "ws/fanout/" + System.nanoTime();
@@ -364,6 +398,7 @@ public class IotMqttWebSocketIntegrationTest {
static WsClient connect(String url, String clientId, String username, String password) throws Exception {
MqttClient client = new MqttClient(url, clientId, new MemoryPersistence());
+ client.setTimeToWait(10_000);
WsClient wsClient = new WsClient(client);
client.setCallback(new MqttCallback() {
@Override