Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(iot): run MQTT topic rules off the event loop (#4462)

Mario Toffia84363f9570b9
diff --git a/docs/services/iot.md b/docs/services/iot.md
index a4314dda2a..10ae4bdbf5 100644
--- a/docs/services/iot.md
+++ b/docs/services/iot.md
@@ -170,6 +170,8 @@ Implementation notes:
- Vert.x MQTT handles the wire protocol and connection lifecycle.
- Floci-owned session, subscription, and retained-message state drives local AWS IoT compatibility behavior.
- Normal client publishes call `IotService.publish(...)` so retained-message storage, event recording, and rule evaluation remain service-owned.
+- An MQTT publish stores the retained message and records the publish on the event loop, then hands topic rule evaluation to a single-thread worker the broker owns, which runs the rules of MQTT publishes one at a time in the order they arrived. A slow rule action therefore delays the rules of later publishes from every connection, but not their PUBACK or fan-out, and not other services' work.
+- A state reset or a broker shutdown skips the rules of MQTT publishes still waiting, and an MQTT publish received during the reset or while the broker is stopped never runs its rules. An evaluation already running finishes.
- Internal broker publishes fan out only to MQTT subscribers and do not recursively evaluate IoT topic rules.
Current accepted limitation:
@@ -177,6 +179,7 @@ Current accepted limitation:
- Certificate and `iot:Connect` checks are enforced on 8883 only; topic-level authorization is not enforced on either port yet.
- Persistent offline sessions are not modeled yet.
- QoS 2 and advanced MQTT 5 property semantics remain follow-up scope.
+- At most 1,000 MQTT publishes are pending rule evaluation, running or waiting. While that many are pending, further publishes are still delivered to subscribers but their topic rules are skipped, with one warning logged until the backlog drains.
## Implementation Shape
@@ -211,9 +214,9 @@ Supported rule behavior:
- Rule SQL parsing and evaluation for the subset described under [Rule SQL](#rule-sql): the `SELECT` projection,
the `FROM` topic filter, and the `WHERE` predicate.
- MQTT-style topic filter matching for exact topics, `+`, and terminal `#`.
-- IoT Data `Publish` and MQTT publishes use the same rule dispatch path.
+- IoT Data `Publish` and MQTT publishes use the same rule dispatch path. An IoT Data `Publish` evaluates its rules before it returns. An MQTT publish evaluates them on the broker's rule worker, so neither its PUBACK nor its fan-out waits for rule actions, and a republished message can reach subscribers before or after the source message's other deliveries; AWS gives no ordering guarantee there either.
- Rule matching is region-scoped: an IoT Data `Publish` evaluates the rules of the region named by its SigV4 credential, and a rule's actions target the rule's own region.
-- Publishes that carry no region — MQTT, or an IoT Data `Publish` whose `Authorization` header is absent or not SigV4 — are evaluated against every region's rules.
+- Publishes that carry no region (MQTT, or an IoT Data `Publish` whose `Authorization` header is absent or not SigV4) are evaluated against every region's rules.
- Actions receive the projected document, which is the payload itself for a statement that selects only `*`.
- `republish` action republishes to another MQTT topic through `IotMqttBrokerService`.
- `sqs` action sends to an SQS queue through Floci's SQS service boundary.
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
index 7f0022da8a..c6330539bc 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.iot;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.services.iot.model.IotRetainedMessage;
import io.netty.handler.codec.mqtt.MqttConnectReturnCode;
import io.netty.handler.codec.mqtt.MqttQoS;
@@ -11,6 +12,7 @@ import io.quarkus.tls.TlsConfiguration;
import io.quarkus.tls.TlsConfigurationRegistry;
import io.quarkus.tls.runtime.config.TlsConfig;
import io.vertx.core.Vertx;
+import io.vertx.core.WorkerExecutor;
import io.vertx.core.buffer.Buffer;
import io.vertx.core.http.ClientAuth;
import io.vertx.core.net.KeyCertOptions;
@@ -19,9 +21,11 @@ import io.vertx.core.net.TrustOptions;
import io.vertx.mqtt.MqttEndpoint;
import io.vertx.mqtt.MqttServer;
import io.vertx.mqtt.MqttServerOptions;
+import io.vertx.mqtt.MqttTopicSubscription;
import io.vertx.mqtt.messages.MqttPublishMessage;
import io.vertx.mqtt.messages.MqttSubscribeMessage;
import io.vertx.mqtt.messages.MqttUnsubscribeMessage;
+import jakarta.annotation.Priority;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.event.Observes;
import jakarta.enterprise.inject.Instance;
@@ -46,10 +50,13 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
@ApplicationScoped
-public class IotMqttBrokerService {
+public class IotMqttBrokerService implements Resettable {
private static final Logger LOG = Logger.getLogger(IotMqttBrokerService.class);
@@ -102,12 +109,27 @@ public class IotMqttBrokerService {
private static final Pattern IPV4_LITERAL = Pattern.compile("\\d{1,3}(\\.\\d{1,3}){3}");
+ static final int MAX_PENDING_RULE_EVALUATIONS = 1_000;
+
+ private static final long RULES_RESETTING = 1;
+ private static final long RULES_STOPPED = 2;
+ private static final long RULE_STATE_STEP = 4;
+
private final EmulatorConfig config;
private final Vertx vertx;
private final Instance<IotService> iotService;
private final TlsConfigurationRegistry tlsRegistry;
+ final WorkerExecutor ruleWorker;
private final Map<String, ClientSession> sessionsByClient = new ConcurrentHashMap<>();
private final Map<String, Map<String, Subscription>> subscriptionsByClient = new ConcurrentHashMap<>();
+ private final AtomicInteger pendingRuleEvaluations = new AtomicInteger();
+ private final AtomicBoolean ruleEvaluationsThrottled = new AtomicBoolean();
+ /**
+ * Rule admission in one value: the {@link #RULES_RESETTING} and {@link #RULES_STOPPED} bits suspend rules, and
+ * every change also adds {@link #RULE_STATE_STEP}, so an evaluation runs its rules only while the state is still
+ * the one it was admitted under.
+ */
+ private final AtomicLong ruleState = new AtomicLong();
private MqttServer server;
private MqttServer tlsServer;
private ReloadingKeyManager keyManager;
@@ -119,6 +141,7 @@ public class IotMqttBrokerService {
this.vertx = vertx;
this.iotService = iotService;
this.tlsRegistry = tlsRegistry;
+ this.ruleWorker = vertx.createSharedWorkerExecutor("floci-iot-mqtt-rules", 1);
}
void onStart(@Observes StartupEvent ignored) {
@@ -133,10 +156,45 @@ public class IotMqttBrokerService {
startIfEnabled();
}
+ // Rules stop before EmulatorLifecycle's default-priority storage flush and shutdown. The blocking server close
+ // stays at default priority, so this ordering does not move it ahead of the flush.
+ void onShutdownStarted(@Observes @Priority(1000) ShutdownEvent ignored) {
+ suspendRules(RULES_STOPPED);
+ }
+
void onStop(@Observes ShutdownEvent ignored) {
stop();
}
+ /**
+ * Emulator reset: rules still waiting are skipped, and publishes received until
+ * {@link #afterReset()} run none, so none writes into the cleared state; one already running
+ * finishes.
+ */
+ @Override
+ public void beforeReset() {
+ suspendRules(RULES_RESETTING);
+ }
+
+ /**
+ * Rules run again for publishes received from now on, unless the broker is stopped. The state
+ * moves on only if {@link #beforeReset()} suspended rules, so an evaluation queued before or
+ * during the reset still skips its rules, and one waiting when a lone afterReset() runs does
+ * not.
+ */
+ @Override
+ public void afterReset() {
+ resumeRules(RULES_RESETTING);
+ }
+
+ /**
+ * The broker keeps nothing in Floci storage; a reset only suspends rules, in
+ * {@link #beforeReset()}.
+ */
+ @Override
+ public void clear() {
+ }
+
synchronized void startIfEnabled() {
if (!config.services().iot().enabled() || !config.services().iot().mqtt().enabled()) {
return;
@@ -145,14 +203,21 @@ public class IotMqttBrokerService {
return;
}
- MqttServer mqttServer = listen(new MqttServerOptions()
- .setHost(config.services().iot().mqtt().host())
- .setPort(config.services().iot().mqtt().port())
- .setMaxMessageSize(MAX_PACKET_SIZE), "IoT MQTT broker", false);
+ // Rules resume before either listener accepts a connection, so a shutdown starting during the listen
+ // keeps them off, and a failed start suspends them again.
+ resumeRules(RULES_STOPPED);
+ MqttServer mqttServer = null;
try {
+ mqttServer = listen(new MqttServerOptions()
+ .setHost(config.services().iot().mqtt().host())
+ .setPort(config.services().iot().mqtt().port())
+ .setMaxMessageSize(MAX_PACKET_SIZE), "IoT MQTT broker", false);
startTlsListener();
} catch (RuntimeException e) {
- mqttServer.close().toCompletionStage().toCompletableFuture().join();
+ suspendRules(RULES_STOPPED);
+ if (mqttServer != null) {
+ mqttServer.close().toCompletionStage().toCompletableFuture().join();
+ }
throw e;
}
server = mqttServer;
@@ -225,6 +290,7 @@ public class IotMqttBrokerService {
}
synchronized void stop() {
+ suspendRules(RULES_STOPPED);
MqttServer mqttServer = server;
MqttServer mqttTlsServer = tlsServer;
if (mqttServer == null && mqttTlsServer == null) {
@@ -251,15 +317,20 @@ public class IotMqttBrokerService {
LOG.info("IoT MQTT broker stopped");
}
+ private void suspendRules(long reason) {
+ ruleState.updateAndGet(state -> (state | reason) + RULE_STATE_STEP);
+ }
+
+ private void resumeRules(long reason) {
+ ruleState.updateAndGet(state -> (state & reason) == 0 ? state : (state & ~reason) + RULE_STATE_STEP);
+ }
+
public synchronized boolean isRunning() {
return server != null;
}
void publish(String topic, byte[] payload) {
- if (server == null) {
- return;
- }
- fanOut(topic, payload == null ? new byte[0] : payload, false);
+ fanOut(topic, payload, false);
}
boolean disconnectClient(String clientId, boolean cleanSession) {
@@ -299,7 +370,7 @@ public class IotMqttBrokerService {
* connection. The plaintext listener, which the WebSocket bridge also lands on, stays open to
* every client.
*/
- private void handleEndpoint(MqttEndpoint endpoint, boolean verifyDevice) {
+ void handleEndpoint(MqttEndpoint endpoint, boolean verifyDevice) {
String clientId = endpoint.clientIdentifier();
SocketAddress remoteAddress = endpoint.remoteAddress();
String sourceIp = remoteAddress == null ? null : remoteAddress.host();
@@ -412,7 +483,7 @@ public class IotMqttBrokerService {
List<MqttQoS> grantedQos = new ArrayList<>();
List<Subscription> accepted = new ArrayList<>();
- for (io.vertx.mqtt.MqttTopicSubscription requested : message.topicSubscriptions()) {
+ for (MqttTopicSubscription requested : message.topicSubscriptions()) {
String topicFilter = requested.topicName();
MqttQoS qos = requested.qualityOfService();
if (!isValidTopicFilter(topicFilter) || qos == MqttQoS.EXACTLY_ONCE) {
@@ -466,17 +537,60 @@ public class IotMqttBrokerService {
return;
}
- iotService.get().publish(topic, payload, message.isRetain(), message.qosLevel().value(), null, session.clientId());
+ iotService.get().publish(topic, payload, message.isRetain(), message.qosLevel().value(), null,
+ session.clientId(), rules -> evaluateRulesOnWorker(topic, rules));
fanOut(topic, payload, false);
}
+ /**
+ * Runs a publish's topic rules on the broker's own single-thread worker, one publish at a time
+ * in the order they were submitted, so a rule action that blocks holds neither an event loop
+ * nor another service's blocking tasks, though it delays the rules of later publishes. At most
+ * {@link #MAX_PENDING_RULE_EVALUATIONS} evaluations are pending, running or waiting; beyond
+ * that a publish's rules are skipped. A reset or a stop skips the rules of publishes still
+ * waiting, and a publish received during the reset or while the broker is stopped never runs
+ * its rules.
+ */
+ void evaluateRulesOnWorker(String topic, Runnable rules) {
+ long admitted = ruleState.get();
+ if ((admitted & (RULES_RESETTING | RULES_STOPPED)) != 0) {
+ return;
+ }
+ // ponytail: beyond the bound a publish's rules are skipped rather than applying backpressure, since QoS 0 has no PUBACK to withhold.
+ if (pendingRuleEvaluations.incrementAndGet() > MAX_PENDING_RULE_EVALUATIONS) {
+ pendingRuleEvaluations.decrementAndGet();
+ if (ruleEvaluationsThrottled.compareAndSet(false, true)) {
+ LOG.warnv("IoT rule evaluation for topic {0} is skipped because {1} evaluations are already pending; later skips are not logged until the backlog drains",
+ topic, Integer.toString(MAX_PENDING_RULE_EVALUATIONS));
+ }
+ return;
+ }
+ ruleWorker.executeBlocking(() -> {
+ if (ruleState.get() == admitted) {
+ rules.run();
+ }
+ return null;
+ }, false).onComplete(result -> {
+ if (pendingRuleEvaluations.decrementAndGet() == 0) {
+ ruleEvaluationsThrottled.set(false);
+ }
+ if (result.failed()) {
+ LOG.warnv(result.cause(), "IoT rule evaluation for topic {0} failed", topic);
+ }
+ });
+ }
+
private void fanOut(String topic, byte[] payload, boolean retained) {
byte[] safePayload = payload == null ? new byte[0] : payload.clone();
for (ClientSession session : sessionsByClient.values()) {
if (!session.endpoint().isConnected() || !hasMatchingSubscription(session.clientId(), topic)) {
continue;
}
- session.endpoint().publish(topic, Buffer.buffer(safePayload), MqttQoS.AT_MOST_ONCE, false, retained);
+ try {
+ session.endpoint().publish(topic, Buffer.buffer(safePayload), MqttQoS.AT_MOST_ONCE, false, retained);
+ } catch (IllegalStateException e) {
+ LOG.debugv("IoT MQTT client {0} closed before the publish on {1} reached it", session.clientId(), topic);
+ }
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java
index 4b14be11fc..2004bc5d78 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java
@@ -72,6 +72,7 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executor;
import java.util.function.BiConsumer;
import java.util.function.Predicate;
import java.util.regex.Pattern;
@@ -884,6 +885,15 @@ public class IotService {
/** {@code clientId} is the MQTT client that published, or null for a message that did not come over MQTT. */
public void publish(String topic, byte[] payload, boolean retain, int qos, String region, String clientId) {
+ publish(topic, payload, retain, qos, region, clientId, Runnable::run);
+ }
+
+ /**
+ * Stores or clears the retained message and records the publish on the calling thread, then
+ * hands the topic rule evaluation to {@code ruleRunner}.
+ */
+ public void publish(String topic, byte[] payload, boolean retain, int qos, String region, String clientId,
+ Executor ruleRunner) {
byte[] eventPayload = payload == null ? new byte[0] : payload;
if (retain) {
if (eventPayload.length == 0) {
@@ -897,7 +907,7 @@ public class IotService {
retainedMessageStore.put(retainedMessageKey(topic), retained);
}
}
- handlePublish(topic, eventPayload, true, region, clientId);
+ handlePublish(topic, eventPayload, true, region, clientId, ruleRunner);
}
public void deleteConnection(String clientId, boolean cleanSession) {
@@ -1281,18 +1291,21 @@ public class IotService {
topicRuleStore.put(topicRuleKey(region, ruleName), rule);
}
- void handlePublish(String topic, byte[] payload, boolean evaluateRules, String region, String clientId) {
+ void handlePublish(String topic, byte[] payload, boolean evaluateRules, String region, String clientId,
+ Executor ruleRunner) {
byte[] eventPayload = payload == null ? new byte[0] : payload;
publishEventRecorder.record(topic, eventPayload);
if (!evaluateRules) {
return;
}
- for (IotTopicRule rule : rulesForPublish(region)) {
- if (!rule.isRuleDisabled()) {
- matchAndProject(rule, topic, clientId, eventPayload)
- .ifPresent(document -> executeTopicRule(rule, topic, eventPayload, document));
+ ruleRunner.execute(() -> {
+ for (IotTopicRule rule : rulesForPublish(region)) {
+ if (!rule.isRuleDisabled()) {
+ matchAndProject(rule, topic, clientId, eventPayload)
+ .ifPresent(document -> executeTopicRule(rule, topic, eventPayload, document));
+ }
}
- }
+ });
}
/**
@@ -1492,7 +1505,7 @@ public class IotService {
case "republish" -> {
String targetTopic = action.path("topic").asText(null);
if (targetTopic != null && !targetTopic.isBlank()) {
- handlePublish(targetTopic, payload, false, region, null);
+ handlePublish(targetTopic, payload, false, region, null, Runnable::run);
mqttBrokerService.publish(targetTopic, payload);
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerServiceTest.java
index 04898b22b0..3ca77e0d25 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerServiceTest.java
@@ -1,18 +1,44 @@
package io.github.hectorvent.floci.services.iot;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.config.FlociCertificateAuthority;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.acm.CertificateGenerator;
import io.github.hectorvent.floci.services.acm.model.KeyAlgorithm;
+import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsService;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbFacade;
+import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbItemAccess;
+import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbOperations.Scope;
+import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbTableAccess;
+import io.github.hectorvent.floci.services.firehose.FirehoseService;
+import io.github.hectorvent.floci.services.iam.IamPolicyEvaluator;
+import io.github.hectorvent.floci.services.kinesis.KinesisService;
+import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.sns.SnsService;
+import io.github.hectorvent.floci.services.sqs.SqsService;
+import io.netty.handler.codec.mqtt.MqttQoS;
import io.quarkus.tls.CertificateUpdatedEvent;
import io.quarkus.tls.TlsConfiguration;
import io.quarkus.tls.TlsConfigurationRegistry;
import io.quarkus.tls.runtime.config.TlsConfig;
+import io.vertx.core.Handler;
import io.vertx.core.Vertx;
import io.vertx.core.buffer.Buffer;
import io.vertx.core.net.KeyCertOptions;
import io.vertx.core.net.PemKeyCertOptions;
+import io.vertx.mqtt.MqttEndpoint;
+import io.vertx.mqtt.MqttTopicSubscription;
+import io.vertx.mqtt.messages.MqttSubscribeMessage;
import jakarta.enterprise.inject.Instance;
+import org.eclipse.paho.client.mqttv3.MqttClient;
+import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
+import org.eclipse.paho.client.mqttv3.MqttException;
+import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
@@ -32,6 +58,7 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.security.KeyStore;
import java.security.Principal;
@@ -42,26 +69,38 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
/**
* The MQTT over TLS listener against a real Vert.x and a mocked TLS registry: what it serves,
* when it is not opened, how a bind failure is reported, and how a reloaded server certificate
- * reaches the next handshake without a restart.
+ * reaches the next handshake without a restart. Also that a publish's topic rules run on the
+ * broker's own single-thread worker, off the event loop.
*/
class IotMqttBrokerServiceTest {
@@ -327,6 +366,412 @@ class IotMqttBrokerServiceTest {
assertEquals(serial(reissuedLeaf), handshake(null).getSerialNumber());
}
+ /**
+ * A rule action that blocks (here a DynamoDB put held on a latch) runs off the event loop: the
+ * publisher gets its PUBACK and a subscriber its copy while the put is still held, and the put
+ * then lands as the rule's owner in the rule's region.
+ */
+ @Test
+ void aBlockedRuleActionHoldsNeitherThePubackNorTheFanOut() throws Exception {
+ CountDownLatch putStarted = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ CountDownLatch putDone = new CountDownLatch(1);
+ AtomicReference<Scope> written = new AtomicReference<>();
+ DynamoDbItemAccess items = mock(DynamoDbItemAccess.class);
+ doAnswer(invocation -> {
+ putStarted.countDown();
+ release.await(10, TimeUnit.SECONDS);
+ written.set(invocation.getArgument(0));
+ putDone.countDown();
+ return null;
+ }).when(items).putItem(any(), any(), any(), any(), any(), any());
+ startBrokerWithRuleWriting(items);
+ BlockingQueue<String> fannedOut = new LinkedBlockingQueue<>();
+ MqttClient subscriber = connectPlain("subscriber");
+ try {
+ subscriber.subscribe("devices/+/metrics", 0,
+ (topic, message) -> fannedOut.add(new String(message.getPayload(), StandardCharsets.UTF_8)));
+ MqttClient publisher = connectPlain("publisher");
+ try {
+ // A QoS 1 publish returns once its PUBACK arrives.
+ publisher.publish("devices/d1/metrics", "{\"t\":1}".getBytes(StandardCharsets.UTF_8), 1, false);
+
+ assertTrue(putStarted.await(10, TimeUnit.SECONDS), "the rule action started");
+ assertEquals("{\"t\":1}", fannedOut.poll(10, TimeUnit.SECONDS), "fanned out while the put is held");
+ assertEquals(1, putDone.getCount(), "the put is still held");
+ } finally {
+ release.countDown();
+ disconnect(publisher);
+ }
+ } finally {
+ disconnect(subscriber);
+ }
+
+ assertTrue(putDone.await(10, TimeUnit.SECONDS));
+ assertEquals(new Scope("111122223333", "eu-west-1"), written.get());
+ }
+
+ /**
+ * A rule action held on a latch occupies only the broker's own worker: an ordered blocking task
+ * of the context that started the broker still runs.
+ */
+ @Test
+ void aHeldRuleActionDoesNotHoldBlockingTasksOfTheContextThatStartedTheBroker() throws Exception {
+ CountDownLatch putStarted = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ DynamoDbItemAccess items = mock(DynamoDbItemAccess.class);
+ doAnswer(invocation -> {
+ putStarted.countDown();
+ release.await(10, TimeUnit.SECONDS);
+ return null;
+ }).when(items).putItem(any(), any(), any(), any(), any(), any());
+ startBrokerWithRuleWriting(items);
+ MqttClient publisher = connectPlain("publisher");
+ try {
+ publisher.publish("devices/d1/metrics", "{\"t\":1}".getBytes(StandardCharsets.UTF_8), 1, false);
+ assertTrue(putStarted.await(10, TimeUnit.SECONDS), "the rule action started");
+
+ assertEquals("ran", vertx.executeBlocking(() -> "ran", true)
+ .toCompletionStage().toCompletableFuture().get(5, TimeUnit.SECONDS),
+ "an ordered blocking task of the starting context ran while the put is held");
+ } finally {
+ release.countDown();
+ disconnect(publisher);
+ }
+ }
+
+ /**
+ * A subscriber whose endpoint closes between the connected check and the write is skipped: the
+ * publish still reaches every other subscriber.
+ */
+ @Test
+ void aSubscriberClosingMidFanOutDoesNotStopTheOthersGettingTheirCopy() throws Exception {
+ startBrokerWithRuleWriting(mock(DynamoDbItemAccess.class));
+ AtomicReference<Handler<MqttSubscribeMessage>> subscribeHandler = new AtomicReference<>();
+ MqttEndpoint closing = mock(MqttEndpoint.class);
+ when(closing.clientIdentifier()).thenReturn("closing");
+ when(closing.isConnected()).thenReturn(true);
+ when(closing.subscribeHandler(any())).thenAnswer(invocation -> {
+ subscribeHandler.set(invocation.getArgument(0));
+ return closing;
+ });
+ when(closing.publish(anyString(), any(), any(), anyBoolean(), anyBoolean()))
+ .thenThrow(new IllegalStateException("the connection closed"));
+ MqttTopicSubscription requested = mock(MqttTopicSubscription.class);
+ when(requested.topicName()).thenReturn("devices/+/metrics");
+ when(requested.qualityOfService()).thenReturn(MqttQoS.AT_MOST_ONCE);
+ MqttSubscribeMessage subscribe = mock(MqttSubscribeMessage.class);
+ when(subscribe.topicSubscriptions()).thenReturn(List.of(requested));
+ broker.handleEndpoint(closing, false);
+ subscribeHandler.get().handle(subscribe);
+
+ BlockingQueue<String> fannedOut = new LinkedBlockingQueue<>();
+ MqttClient subscriber = connectPlain("subscriber");
+ try {
+ subscriber.subscribe("devices/+/metrics", 0,
+ (topic, message) -> fannedOut.add(new String(message.getPayload(), StandardCharsets.UTF_8)));
+
+ broker.publish("devices/d1/metrics", "{\"t\":1}".getBytes(StandardCharsets.UTF_8));
+
+ verify(closing).publish(eq("devices/d1/metrics"), any(), any(), anyBoolean(), anyBoolean());
+ assertEquals("{\"t\":1}", fannedOut.poll(10, TimeUnit.SECONDS), "the other subscriber got its copy");
+ } finally {
+ disconnect(subscriber);
+ }
+ }
+
+ /**
+ * Two publishes from two connections, the first one's rule action held on a latch: the second
+ * publish is fanned out, so its rules were handed over, yet they start only once the first
+ * publish's rules finished.
+ */
+ @Test
+ void rulesOfALaterPublishWaitForTheEarlierPublishToFinish() throws Exception {
+ List<String> order = new CopyOnWriteArrayList<>();
+ CountDownLatch firstStarted = new CountDownLatch(1);
+ CountDownLatch releaseFirst = new CountDownLatch(1);
+ CountDownLatch secondStarted = new CountDownLatch(1);
+ CountDownLatch secondDone = new CountDownLatch(1);
+ DynamoDbItemAccess items = mock(DynamoDbItemAccess.class);
+ doAnswer(invocation -> {
+ String publish = invocation.<JsonNode>getArgument(2).path("t").path("N").asText();
+ order.add(publish + "-start");
+ if (publish.equals("1")) {
+ firstStarted.countDown();
+ releaseFirst.await(10, TimeUnit.SECONDS);
+ } else {
+ secondStarted.countDown();
+ }
+ order.add(publish + "-end");
+ if (publish.equals("2")) {
+ secondDone.countDown();
+ }
+ return null;
+ }).when(items).putItem(any(), any(), any(), any(), any(), any());
+ startBrokerWithRuleWriting(items);
+ BlockingQueue<String> fannedOut = new LinkedBlockingQueue<>();
+ MqttClient subscriber = connectPlain("subscriber");
+ try {
+ subscriber.subscribe("devices/+/metrics", 0,
+ (topic, message) -> fannedOut.add(new String(message.getPayload(), StandardCharsets.UTF_8)));
+ MqttClient publisher = connectPlain("publisher");
+ MqttClient publisher2 = connectPlain("publisher2");
+ try {
+ publisher.publish("devices/d1/metrics", "{\"t\":1}".getBytes(StandardCharsets.UTF_8), 1, false);
+ publisher2.publish("devices/d1/metrics", "{\"t\":2}".getBytes(StandardCharsets.UTF_8), 1, false);
+
+ assertTrue(firstStarted.await(10, TimeUnit.SECONDS), "the first publish's rules started");
+ assertEquals("{\"t\":1}", fannedOut.poll(10, TimeUnit.SECONDS));
+ assertEquals("{\"t\":2}", fannedOut.poll(10, TimeUnit.SECONDS), "the second publish was fanned out");
+ assertFalse(secondStarted.await(300, TimeUnit.MILLISECONDS), "the second waits while the first is held");
+ order.add("released");
+ } finally {
+ releaseFirst.countDown();
+ disconnect(publisher);
+ disconnect(publisher2);
+ }
+ } finally {
+ disconnect(subscriber);
+ }
+
+ assertTrue(secondDone.await(10, TimeUnit.SECONDS), "the second publish's rules ran");
+ assertEquals(List.of("1-start", "released", "1-end", "2-start", "2-end"), order);
+ }
+
+ /**
+ * One evaluation held on a latch and the rest of the bound queued behind it: the rules of the
+ * next publish are skipped, and once the backlog drains a later publish's rules run again.
+ */
+ @Test
+ void rulesOfAPublishBeyondThePendingBoundAreSkippedUntilTheBacklogDrains() throws Exception {
+ String topic = "devices/d1/metrics";
+ int queued = IotMqttBrokerService.MAX_PENDING_RULE_EVALUATIONS - 1;
+ CountDownLatch firstStarted = new CountDownLatch(1);
+ CountDownLatch releaseFirst = new CountDownLatch(1);
+ AtomicInteger queuedRan = new AtomicInteger();
+ AtomicBoolean beyondBoundRan = new AtomicBoolean();
+ CountDownLatch afterDrainRan = new CountDownLatch(1);
+
+ broker.evaluateRulesOnWorker(topic, () -> {
+ firstStarted.countDown();
+ try {
+ releaseFirst.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("interrupted while the first evaluation was held", e);
+ }
+ });
+ try {
+ assertTrue(firstStarted.await(10, TimeUnit.SECONDS), "the first evaluation started");
+ for (int i = 0; i < queued; i++) {
+ broker.evaluateRulesOnWorker(topic, queuedRan::incrementAndGet);
+ }
+ broker.evaluateRulesOnWorker(topic, () -> beyondBoundRan.set(true));
+ } finally {
+ releaseFirst.countDown();
+ }
+
+ long deadline = System.currentTimeMillis() + 10_000;
+ while (queuedRan.get() < queued) {
+ if (System.currentTimeMillis() > deadline) {
+ throw new AssertionError("only " + queuedRan.get() + " of " + queued + " queued evaluations ran");
+ }
+ Thread.sleep(10);
+ }
+ assertFalse(beyondBoundRan.get(), "the evaluation beyond the bound was skipped");
+
+ broker.evaluateRulesOnWorker(topic, afterDrainRan::countDown);
+ assertTrue(afterDrainRan.await(10, TimeUnit.SECONDS), "capacity came back once the backlog drained");
+ assertFalse(beyondBoundRan.get(), "the evaluation beyond the bound never ran");
+ }
+
+ @Test
+ void rulesAreSkippedFromAResetUntilItEnds() throws Exception {
+ assertSkipsWaitingAndLaterRules(broker::beforeReset, broker::afterReset);
+ assertTrue(rulesRun(), "rules run again once the reset ends");
+ }
+
+ @Test
+ void rulesAreSkippedFromAStopUntilARestart() throws Exception {
+ broker.startIfEnabled();
+ assertSkipsWaitingAndLaterRules(broker::stop, broker::startIfEnabled);
+ assertTrue(rulesRun(), "a publish after the restart runs its rules");
+ }
+
+ @Test
+ void rulesAreSkippedFromTheShutdownObserverUntilARestart() throws Exception {
+ broker.startIfEnabled();
+ assertSkipsWaitingAndLaterRules(() -> broker.onShutdownStarted(null), () -> {
+ broker.stop();
+ broker.startIfEnabled();
+ });
+ assertTrue(rulesRun(), "a publish after the restart runs its rules");
+ }
+
+ @Test
+ void aStartDuringAResetDoesNotResumeRules() throws Exception {
+ broker.beforeReset();
+ broker.startIfEnabled();
+ assertFalse(rulesRun(), "a start during the reset leaves rules suspended");
+
+ broker.afterReset();
+ assertTrue(rulesRun(), "rules run again once the reset ends");
+ }
+
+ @Test
+ void aResetEndingAfterShutdownDoesNotResumeRules() throws Exception {
+ broker.startIfEnabled();
+ broker.beforeReset();
+ broker.stop();
+ broker.afterReset();
+ assertFalse(rulesRun(), "the end of the reset leaves the stopped broker's rules suspended");
+ }
+
+ /**
+ * The reset controller calls afterReset() on every service, even one whose beforeReset() never
+ * ran; such a lone afterReset() leaves the rule state alone, so an evaluation waiting then
+ * still runs its rules.
+ */
+ @Test
+ void anAfterResetWithoutABeforeResetKeepsWaitingRules() throws Exception {
+ String topic = "devices/d1/metrics";
+ CountDownLatch started = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicBoolean waitingRan = new AtomicBoolean();
+
+ broker.evaluateRulesOnWorker(topic, () -> {
+ started.countDown();
+ try {
+ release.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("interrupted while the first evaluation was held", e);
+ }
+ });
+ try {
+ assertTrue(started.await(10, TimeUnit.SECONDS), "the first evaluation started");
+ broker.evaluateRulesOnWorker(topic, () -> waitingRan.set(true));
+ broker.afterReset();
+ } finally {
+ release.countDown();
+ }
+
+ broker.ruleWorker.executeBlocking(() -> null, false).toCompletionStage().toCompletableFuture().get(10, TimeUnit.SECONDS);
+ assertTrue(waitingRan.get(), "the evaluation waiting at the lone afterReset() still ran");
+ }
+
+ /**
+ * A restart whose TLS listener cannot bind fails and leaves rules off; a later successful start
+ * turns them on again.
+ */
+ @Test
+ void aFailedRestartKeepsRulesOff() throws Exception {
+ broker.startIfEnabled();
+ broker.stop();
+ try (ServerSocket blocker = new ServerSocket()) {
+ blocker.bind(new InetSocketAddress("127.0.0.1", tlsPort));
+
+ assertThrows(IllegalStateException.class, broker::startIfEnabled);
+ assertFalse(rulesRun(), "a failed restart leaves rules off");
+ }
+
+ broker.startIfEnabled();
+ assertTrue(rulesRun(), "a later successful start turns rules on again");
+ }
+
+ /**
+ * One evaluation held on a latch, one queued before {@code suspend} runs and one submitted after
+ * it, which is refused at admission, then {@code resume} ends the suspension while the queued
+ * one still waits: the held evaluation finishes, the queued one skips its rules and the refused
+ * one never runs. The barrier is a task on the broker's single-thread rule worker, which runs
+ * tasks in submission order, so it completes only after the evaluations before it.
+ */
+ private void assertSkipsWaitingAndLaterRules(Runnable suspend, Runnable resume) throws Exception {
+ String topic = "devices/d1/metrics";
+ CountDownLatch started = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicBoolean firstFinished = new AtomicBoolean();
+ AtomicBoolean waitingRan = new AtomicBoolean();
+ AtomicBoolean laterRan = new AtomicBoolean();
+
+ broker.evaluateRulesOnWorker(topic, () -> {
+ started.countDown();
+ try {
+ release.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("interrupted while the first evaluation was held", e);
+ }
+ firstFinished.set(true);
+ });
+ try {
+ assertTrue(started.await(10, TimeUnit.SECONDS), "the first evaluation started");
+ broker.evaluateRulesOnWorker(topic, () -> waitingRan.set(true));
+ suspend.run();
+ broker.evaluateRulesOnWorker(topic, () -> laterRan.set(true));
+ resume.run();
+ } finally {
+ release.countDown();
+ }
+
+ broker.ruleWorker.executeBlocking(() -> null, false).toCompletionStage().toCompletableFuture().get(10, TimeUnit.SECONDS);
+ assertTrue(firstFinished.get(), "the running evaluation finished");
+ assertFalse(waitingRan.get(), "the waiting evaluation was skipped");
+ assertFalse(laterRan.get(), "the evaluation submitted during the suspension was refused");
+ }
+
+ /** Submits one evaluation and reports whether its rules ran, once the broker's rule worker processed it. */
+ private boolean rulesRun() throws Exception {
+ AtomicBoolean ran = new AtomicBoolean();
+ broker.evaluateRulesOnWorker("devices/d1/metrics", () -> ran.set(true));
+ broker.ruleWorker.executeBlocking(() -> null, false).toCompletionStage().toCompletableFuture().get(10, TimeUnit.SECONDS);
+ return ran.get();
+ }
+
+ /** Starts the broker over a real {@link IotService} whose one rule writes every metrics publish to DynamoDB. */
+ private void startBrokerWithRuleWriting(DynamoDbItemAccess items) throws Exception {
+ StorageFactory storageFactory = mock(StorageFactory.class);
+ when(storageFactory.create(anyString(), anyString(), any()))
+ .thenAnswer(invocation -> AccountAwareStorageBackend.inMemory("000000000000"));
+ EmulatorConfig serviceConfig = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(serviceConfig.defaultRegion()).thenReturn("us-east-1");
+ when(serviceConfig.defaultAccountId()).thenReturn("000000000000");
+ when(serviceConfig.services().iot().ruleSqlStrict()).thenReturn(false);
+ ObjectMapper mapper = new ObjectMapper();
+ IotService service = new IotService(storageFactory, serviceConfig, new RegionResolver("eu-west-1", "111122223333"),
+ mapper, new IotPublishEventRecorder(), broker, mock(SqsService.class), mock(SnsService.class),
+ mock(S3Service.class), mock(KinesisService.class),
+ new DynamoDbFacade(items, mock(DynamoDbTableAccess.class), new RegionResolver("us-east-1", "000000000000")),
+ mock(LambdaService.class), mock(FirehoseService.class), mock(CloudWatchLogsService.class), ca,
+ new IamPolicyEvaluator(mapper));
+ when(iotService.get()).thenReturn(service);
+ service.createTopicRule("toTable", mapper.readTree("""
+ {"sql": "SELECT * FROM 'devices/+/metrics'",
+ "actions": [{"dynamoDBv2": {"putItem": {"tableName": "metrics"}, "roleArn": "r"}}]}
+ """), "eu-west-1");
+ broker.startIfEnabled();
+ }
+
+ private MqttClient connectPlain(String clientId) throws MqttException {
+ MqttClient client = new MqttClient("tcp://127.0.0.1:" + plainPort, clientId, new MemoryPersistence());
+ MqttConnectOptions options = new MqttConnectOptions();
+ options.setConnectionTimeout(10);
+ options.setAutomaticReconnect(false);
+ client.connect(options);
+ client.setTimeToWait(10_000);
+ return client;
+ }
+
+ private static void disconnect(MqttClient client) throws MqttException {
+ try {
+ if (client.isConnected()) {
+ client.disconnect();
+ }
+ } finally {
+ client.close();
+ }
+ }
+
private X509Certificate handshake(KeyManager[] clientKeys) throws Exception {
try (SSLSocket socket = connectTls(clientKeys)) {
socket.startHandshake();
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java
index 0ddbf43887..bacc529b2a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java
@@ -52,6 +52,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
/**
@@ -75,6 +76,7 @@ class IotServiceTest {
private final DynamoDbItemAccess dynamoDb = mock(DynamoDbItemAccess.class);
private final FirehoseService firehose = mock(FirehoseService.class);
private final CloudWatchLogsService logs = mock(CloudWatchLogsService.class);
+ private final IotPublishEventRecorder recorder = new IotPublishEventRecorder();
private IotService service;
@BeforeEach
@@ -99,7 +101,7 @@ class IotServiceTest {
config,
new RegionResolver(REGION, ACCOUNT),
mapper,
- new IotPublishEventRecorder(),
+ recorder,
mock(IotMqttBrokerService.class),
sqs,
mock(SnsService.class),
@@ -142,7 +144,7 @@ class IotServiceTest {
}
private void publish(String payload) {
- service.handlePublish(TOPIC, payload.getBytes(StandardCharsets.UTF_8), true, REGION, null);
+ service.handlePublish(TOPIC, payload.getBytes(StandardCharsets.UTF_8), true, REGION, null, Runnable::run);
}
private JsonNode capturedInvocationPayload(String functionArn) throws Exception {
@@ -151,6 +153,35 @@ class IotServiceTest {
return mapper.readTree(payload.getValue());
}
+ @Test
+ void publishRecordsAndStoresTheRetainedMessageAtOnceButRunsTheRulesOnlyOnTheRuleRunner() throws Exception {
+ createRule("metricsRule", sqsThenLambdaRule(null));
+ List<Runnable> deferred = new ArrayList<>();
+ byte[] payload = "{\"v\":1}".getBytes(StandardCharsets.UTF_8);
+
+ service.publish(TOPIC, payload, true, 1, null, "sensor-1", deferred::add);
+
+ assertEquals(TOPIC, recorder.recentEvents().get(0).topic());
+ assertEquals(Base64.getEncoder().encodeToString(payload), service.getRetainedMessage(TOPIC).getPayload());
+ verifyNoInteractions(sqs, lambda);
+ assertEquals(1, deferred.size());
+
+ deferred.get(0).run();
+
+ verify(sqs).sendMessage(eq(QUEUE_URL), eq("{\"v\":1}"), eq(0), eq(REGION));
+ verify(lambda).invoke(eq(REGION), eq(FUNCTION_ARN), any(), eq(InvocationType.Event));
+ }
+
+ @Test
+ void publishWithoutARuleRunnerRunsTheRuleActionsBeforeItReturns() throws Exception {
+ createRule("metricsRule", sqsThenLambdaRule(null));
+
+ service.publish(TOPIC, "{\"v\":1}".getBytes(StandardCharsets.UTF_8), false, 0, null, null);
+
+ verify(sqs).sendMessage(eq(QUEUE_URL), eq("{\"v\":1}"), eq(0), eq(REGION));
+ verify(lambda).invoke(eq(REGION), eq(FUNCTION_ARN), any(), eq(InvocationType.Event));
+ }
+
@Test
void aFailingActionDoesNotFailThePublishOrStopTheOtherActions() throws Exception {
createRule("metricsRule", sqsThenLambdaRule(null));
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlIntegrationTest.java
index 803c87ad3a..32b54852aa 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlIntegrationTest.java
@@ -155,7 +155,8 @@ class IotTopicRuleSqlIntegrationTest {
void clientidIsTheMqttClientWhenThePublishCarriesOne() {
createRule("sqlClientIdRule", "SELECT clientid() AS client FROM 'sqltest/clientid/+'", "sqltest/clientid-out");
- iotService.handlePublish("sqltest/clientid/a", "{}".getBytes(StandardCharsets.UTF_8), true, REGION, "sensor-9");
+ iotService.handlePublish("sqltest/clientid/a", "{}".getBytes(StandardCharsets.UTF_8), true, REGION, "sensor-9",
+ Runnable::run);
assertEquals(List.of("{\"client\":\"sensor-9\"}"), republished("sqltest/clientid-out"));
}
@@ -225,7 +226,7 @@ class IotTopicRuleSqlIntegrationTest {
}
private void publish(String topic, String payload) {
- iotService.handlePublish(topic, payload.getBytes(StandardCharsets.UTF_8), true, REGION, null);
+ iotService.handlePublish(topic, payload.getBytes(StandardCharsets.UTF_8), true, REGION, null, Runnable::run);
}
private List<String> republished(String targetTopic) {