Light, fluffy, and always free - The AWS Local Emulator alternative
feat(iot): rule SQL message functions, isNull, isUndefined, IN and array literals (#3036)
Mario Toffiab88034cd1df6
diff --git a/docs/services/iot.md b/docs/services/iot.mdindex c23065db1d..b4f29e1271 100644--- a/docs/services/iot.md+++ b/docs/services/iot.md@@ -219,10 +219,12 @@ statement := SELECT item (',' item)* FROM '<topic filter>' [WHERE expr]item := '*' | operand [AS identifier]expr := term (OR term)*term := factor (AND factor)*-factor := NOT factor | '(' expr ')' | operand [comparison operand]+factor := NOT factor | '(' expr ')' | operand [comparison operand | IN operand]comparison := '=' | '<>' | '!=' | '<' | '<=' | '>' | '>='-operand := path | literal | call+operand := path | literal | array | call+array := '[' [operand (',' operand)*] ']'call := topic() | topic(<segment>) | startswith(operand, operand) | endswith(operand, operand)+ | clientid() | timestamp() | accountid() | newuuid() | isNull(operand) | isUndefined(operand)path := identifier ('.' identifier)*literal := 'string' | "string" | number | TRUE | FALSE | NULL```@@ -231,6 +233,16 @@ Semantics:- Keywords and function names are case insensitive, field names are case sensitive.- `topic()` is the full MQTT topic, `topic(n)` is its nth segment counting from 1.+- `clientid()` is the MQTT client that published the message, or `n/a` for an IoT Data `Publish`+ over HTTP, as on AWS. `accountid()` is the account that owns the rule. `timestamp()` is the+ current time in milliseconds since the epoch. `newuuid()` is a fresh random UUID.+- `isNull(x)` is true only for a JSON `null`, `isUndefined(x)` only for a missing field or an+ undefined expression. Neither is ever undefined itself.+- `x IN y` is true when the array `y` holds a value equal to `x` under the equality rules below,+ false otherwise. It is undefined when `x` is undefined or `y` is not an array. `y` may be a+ payload field or an array literal such as `[1, 'a']`.+- An array literal is undefined as a whole when any of its elements is, so a position is never+ silently dropped.- A select item without `AS` is written under the last segment of its path, or under the functionname, so `topic()` becomes `topic`.- When `*` is present, every payload field is copied first and the other select items are written@@ -244,9 +256,10 @@ Semantics:`clientToken <> 'x'`.- JSON `null` is a value, not `Undefined`: it equals only `NULL`, so `clientToken <> 'x'` is truewhen the field is null and undefined when it is missing.-- `=` and `<>` compare two numbers by value and anything else by type and value, so operands of- different types are simply not equal: `level = '3'` is false and `level <> '3'` is true when- `level` is the number 3. These follow the operator tables in the AWS IoT SQL reference.+- `=` and `<>` compare two numbers by value, arrays element by element in order, objects by key in+ any order, and anything else by type and value, so operands of different types are simply not+ equal: `level = '3'` is false and `level <> '3'` is true when `level` is the number 3. These+ follow the operator tables in the AWS IoT SQL reference.- `<`, `<=`, `>` and `>=` convert both operands to a number. A string converts when it looks likeone (`'10' > 9` is true); any other operand makes the comparison undefined.- Payload numbers are read exactly, never through a double, so `9007199254740993.0`, `1e-400` and@@ -269,10 +282,11 @@ statement with `SqlParseException` instead, the way AWS does. It is `false` by dCurrent limitations:-- Not evaluated: `clientid()`, `timestamp()`, `accountid()`, `principal()`, `newuuid()`, `isNull()`,- `isUndefined()`, `encode()`, `get_thing_shadow()`, arithmetic, array indexing, `IN`, `CASE`, and- `${}` substitution templates in action fields. A rule using any of them takes the unparsed path- described above.+- Not evaluated: `principal()`, which needs the certificate a device authenticated with and the+ broker does not verify one yet; `traceid()`; `encode()`, `get()`, `get_thing_shadow()` and the+ rest of the AWS function list; arithmetic; array indexing; `CASE`; `SELECT VALUE`; object+ literals; the `SET` clause; `EXISTS` subqueries; and `${}` substitution templates in action+ fields. A rule using any of them takes the unparsed path described above.- `awsIotSqlVersion` is stored and echoed back but not acted on. The versions differ in how`SELECT *` treats arrays, which Floci does not model.- Less common AWS IoT rule action types are follow-up scope.diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotDataController.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotDataController.javaindex 7613b26e43..c347acc3dd 100644--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotDataController.java+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotDataController.java@@ -91,7 +91,7 @@ public class IotDataController {byte[] payload) {// Unsigned or non-SigV4 publishes carry no usable region; both count as unresolvedString region = regionResolver.resolveRegionFromAuthOrNull(headers.getHeaderString("Authorization"));- iotService.publish(topic, payload == null ? new byte[0] : payload, Boolean.TRUE.equals(retain), qos == null ? 0 : qos, region);+ iotService.publish(topic, payload == null ? new byte[0] : payload, Boolean.TRUE.equals(retain), qos == null ? 0 : qos, region, null);return Response.ok(objectMapper.createObjectNode()).build();}diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.javaindex 37c05b7878..52c167fcc8 100644--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java@@ -446,7 +446,7 @@ public class IotMqttBrokerService {return;}- iotService.get().publish(topic, payload, message.isRetain(), message.qosLevel().value(), null);+ iotService.get().publish(topic, payload, message.isRetain(), message.qosLevel().value(), null, session.clientId());fanOut(topic, payload, false);}diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotService.javaindex 442cf11482..9e3581c22c 100644--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java@@ -30,6 +30,7 @@ import io.github.hectorvent.floci.services.iot.model.IotTopicRule;import io.github.hectorvent.floci.services.acm.CertificateGenerator;import io.github.hectorvent.floci.services.iot.model.Thing;import io.github.hectorvent.floci.services.iot.rules.RuleSql;+import io.github.hectorvent.floci.services.iot.rules.RuleSqlContext;import io.github.hectorvent.floci.services.iot.rules.RuleSqlEvaluator;import io.github.hectorvent.floci.services.iot.rules.RuleSqlParseException;import io.github.hectorvent.floci.services.iot.rules.RuleSqlParser;@@ -52,6 +53,7 @@ import java.security.MessageDigest;import java.security.NoSuchAlgorithmException;import java.security.cert.CertificateEncodingException;import java.security.cert.X509Certificate;+import java.time.Clock;import java.time.Instant;import java.util.ArrayList;import java.util.Arrays;@@ -214,7 +216,7 @@ public class IotService {this.cloudWatchLogsService = cloudWatchLogsService;this.certificateAuthority = certificateAuthority;this.policyEvaluator = policyEvaluator;- this.ruleSqlEvaluator = new RuleSqlEvaluator(objectMapper);+ this.ruleSqlEvaluator = new RuleSqlEvaluator(objectMapper, Clock.systemUTC());}public String describeEndpoint(String endpointType) {@@ -876,10 +878,11 @@ public class IotService {}public void publish(String topic, byte[] payload) {- publish(topic, payload, false, 0, null);+ publish(topic, payload, false, 0, null, null);}- public void publish(String topic, byte[] payload, boolean retain, int qos, String region) {+ /** {@code clientId} is the MQTT client that published, or null for a message that did not come over MQTT. */+ public void publish(String topic, byte[] payload, boolean retain, int qos, String region, String clientId) {byte[] eventPayload = payload == null ? new byte[0] : payload;if (retain) {if (eventPayload.length == 0) {@@ -893,7 +896,7 @@ public class IotService {retainedMessageStore.put(retainedMessageKey(topic), retained);}}- handlePublish(topic, eventPayload, true, region);+ handlePublish(topic, eventPayload, true, region, clientId);}public void deleteConnection(String clientId, boolean cleanSession) {@@ -1277,7 +1280,7 @@ public class IotService {topicRuleStore.put(topicRuleKey(region, ruleName), rule);}- void handlePublish(String topic, byte[] payload, boolean evaluateRules, String region) {+ void handlePublish(String topic, byte[] payload, boolean evaluateRules, String region, String clientId) {byte[] eventPayload = payload == null ? new byte[0] : payload;publishEventRecorder.record(topic, eventPayload);if (!evaluateRules) {@@ -1285,7 +1288,7 @@ public class IotService {}for (IotTopicRule rule : rulesForPublish(region)) {if (!rule.isRuleDisabled()) {- matchAndProject(rule, topic, eventPayload)+ matchAndProject(rule, topic, clientId, eventPayload).ifPresent(document -> executeTopicRule(rule, topic, eventPayload, document));}}@@ -1296,7 +1299,7 @@ public class IotService {* fire for this message. Rules whose SQL could not be parsed take the pre-parser path: the* topic filter is read straight out of the SQL and the payload is forwarded untouched.*/- private Optional<byte[]> matchAndProject(IotTopicRule rule, String topic, byte[] payload) {+ private Optional<byte[]> matchAndProject(IotTopicRule rule, String topic, String clientId, byte[] payload) {RuleSql query = ruleQuery(rule);if (query == null) {return topicMatches(extractTopicPattern(rule.getSql()), topic) ? Optional.of(payload) : Optional.empty();@@ -1304,7 +1307,9 @@ public class IotService {if (!topicMatches(query.topicFilter(), topic)) {return Optional.empty();}- return ruleSqlEvaluator.evaluate(rule.getRuleName(), query, topic, payload);+ RuleSqlContext context = new RuleSqlContext(topic, clientId,+ AwsArnUtils.accountOrDefault(rule.getRuleArn(), config.defaultAccountId()));+ return ruleSqlEvaluator.evaluate(rule.getRuleName(), query, context, payload);}/**@@ -1486,7 +1491,7 @@ public class IotService {case "republish" -> {String targetTopic = action.path("topic").asText(null);if (targetTopic != null && !targetTopic.isBlank()) {- handlePublish(targetTopic, payload, false, region);+ handlePublish(targetTopic, payload, false, region, null);mqttBrokerService.publish(targetTopic, payload);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSql.java b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSql.javaindex 8d09b1b7a5..0b426a3d78 100644--- a/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSql.java+++ b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSql.java@@ -56,7 +56,8 @@ public record RuleSql(List<Projection> projections, String topicFilter, Expr whepublic record Aliased(Expr expression, String alias) implements Projection {}- public sealed interface Expr permits Path, Literal, Call, Comparison, Conjunction, Disjunction, Negation {+ public sealed interface Expr+ permits Path, Literal, Call, ArrayLiteral, Comparison, Membership, Conjunction, Disjunction, Negation {}/** A dotted field path such as {@code state.reported.temperature}. */@@ -69,16 +70,27 @@ public record RuleSql(List<Projection> projections, String topicFilter, Expr whepublic record Literal(JsonNode value) implements Expr {}- /** A supported function call: {@code topic}, {@code startswith} or {@code endswith}. */+ /** A supported function call under the name AWS documents, such as {@code topic} or {@code isNull}. */public record Call(String function, List<Expr> arguments) implements Expr {public Call {arguments = List.copyOf(arguments);}}+ /** A JSON array written in the statement, such as {@code [lat, long]}. */+ public record ArrayLiteral(List<Expr> elements) implements Expr {+ public ArrayLiteral {+ elements = List.copyOf(elements);+ }+ }+public record Comparison(Expr left, Operator operator, Expr right) implements Expr {}+ /** {@code element IN array}: true when the array holds a value equal to the element. */+ public record Membership(Expr element, Expr array) implements Expr {+ }+public record Conjunction(Expr left, Expr right) implements Expr {}diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlContext.java b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlContext.javanew file mode 100644index 0000000000..a137162aa9--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlContext.java@@ -0,0 +1,9 @@+package io.github.hectorvent.floci.services.iot.rules;++/**+ * What a rule can ask about the message beyond its payload: the topic it was published on,+ * the MQTT client that sent it ({@code null} when it did not arrive over MQTT, which+ * {@code clientid()} reports as {@code n/a} as AWS does), and the account that owns the rule.+ */+public record RuleSqlContext(String topic, String clientId, String accountId) {+}diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluator.java b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluator.javaindex 4189295ad8..7e6fc5e33c 100644--- a/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluator.java+++ b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluator.java@@ -4,17 +4,21 @@ import com.fasterxml.jackson.databind.DeserializationFeature;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.ObjectReader;+import com.fasterxml.jackson.databind.node.ArrayNode;import com.fasterxml.jackson.databind.node.BooleanNode;import com.fasterxml.jackson.databind.node.JsonNodeFactory;+import com.fasterxml.jackson.databind.node.LongNode;import com.fasterxml.jackson.databind.node.ObjectNode;import com.fasterxml.jackson.databind.node.TextNode;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Aliased;+import io.github.hectorvent.floci.services.iot.rules.RuleSql.ArrayLiteral;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Call;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Comparison;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Conjunction;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Disjunction;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Expr;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Literal;+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Membership;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Negation;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Operator;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Path;@@ -25,18 +29,21 @@ import org.jboss.logging.Logger;import java.io.IOException;import java.math.BigDecimal;import java.nio.charset.StandardCharsets;+import java.time.Clock;+import java.util.Map;import java.util.Optional;+import java.util.UUID;import java.util.regex.Pattern;/*** Evaluates a parsed topic rule statement against one published message, following the- * operator and conversion tables of the AWS IoT SQL reference.+ * operator, function and conversion tables of the AWS IoT SQL reference.** <p>A value AWS reports as {@code Undefined} (a missing field, an out of range topic segment,* a function argument that cannot be converted) is a {@code null} {@link JsonNode}. It spreads:- * a comparison, {@code AND}, {@code OR} or {@code NOT} with an undefined operand is undefined,- * and a rule fires only when its {@code WHERE} is true. JSON {@code null} is not undefined: it- * is a value that equals only itself.+ * a comparison, {@code IN}, {@code AND}, {@code OR} or {@code NOT} with an undefined operand is+ * undefined, and a rule fires only when its {@code WHERE} is true. JSON {@code null} is not+ * undefined: it is a value that equals only itself.** <p>Payload numbers are read as {@code BigDecimal}, never through a {@code double}, so they* compare exactly at any size or precision, as AWS's Decimal does.@@ -48,34 +55,40 @@ public final class RuleSqlEvaluator {/** The string forms AWS converts to a number when an ordering operator meets a string. */private static final Pattern NUMERIC_STRING = Pattern.compile("-?\\d+(\\.\\d+)?([eE]-?\\d+)?");+ /** What {@code clientid()} returns for a message that did not arrive over MQTT, as AWS documents. */+ private static final String NO_MQTT_CLIENT = "n/a";+private final ObjectMapper objectMapper;private final ObjectReader payloadReader;+ private final Clock clock;- public RuleSqlEvaluator(ObjectMapper objectMapper) {+ public RuleSqlEvaluator(ObjectMapper objectMapper, Clock clock) {this.objectMapper = objectMapper;this.payloadReader = objectMapper.reader().with(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS).with(JsonNodeFactory.withExactBigDecimals(true));+ this.clock = clock;}/*** Returns the document the rule's actions should receive, or empty when the rule does* not fire for this message.*/- public Optional<byte[]> evaluate(String ruleName, RuleSql query, String topic, byte[] payload) {+ public Optional<byte[]> evaluate(String ruleName, RuleSql query, RuleSqlContext context, byte[] payload) {byte[] message = payload == null ? new byte[0] : payload;if (query.isSelectAllOnly() && query.where() == null) {return Optional.of(message);}ObjectNode document = readObject(message);if (document == null) {- LOG.debugv("Topic rule {0} skipped a payload that is not a JSON object on topic {1}", ruleName, topic);+ LOG.debugv("Topic rule {0} skipped a payload that is not a JSON object on topic {1}",+ ruleName, context.topic());return Optional.empty();}- if (query.where() != null && !Boolean.TRUE.equals(truth(value(query.where(), topic, document)))) {+ if (query.where() != null && !Boolean.TRUE.equals(truth(value(query.where(), context, document)))) {return Optional.empty();}- return Optional.of(query.isSelectAllOnly() ? message : project(query, topic, document));+ return Optional.of(query.isSelectAllOnly() ? message : project(query, context, document));}private ObjectNode readObject(byte[] payload) {@@ -96,14 +109,14 @@ public final class RuleSqlEvaluator {* present, so a select item whose alias collides with a payload field overwrites it,* which is what AWS does for statements such as {@code SELECT *, topic() as topic}.*/- private byte[] project(RuleSql query, String topic, ObjectNode document) {+ private byte[] project(RuleSql query, RuleSqlContext context, ObjectNode document) {ObjectNode projected = objectMapper.createObjectNode();if (query.projections().stream().anyMatch(SelectAll.class::isInstance)) {projected.setAll(document);}for (Projection projection : query.projections()) {if (projection instanceof Aliased aliased) {- JsonNode value = value(aliased.expression(), topic, document);+ JsonNode value = value(aliased.expression(), context, document);if (value != null) {projected.set(aliased.alias(), value);}@@ -112,20 +125,24 @@ public final class RuleSqlEvaluator {return projected.toString().getBytes(StandardCharsets.UTF_8);}- private JsonNode value(Expr expression, String topic, ObjectNode document) {+ private JsonNode value(Expr expression, RuleSqlContext context, ObjectNode document) {return switch (expression) {case Literal literal -> literal.value();case Path path -> resolve(path, document);- case Call call -> call(call, topic, document);- case Comparison comparison -> compare(comparison, topic, document);+ case Call call -> call(call, context, document);+ case ArrayLiteral array -> array(array, context, document);+ case Comparison comparison -> compare(comparison, context, document);+ case Membership membership -> contains(+ value(membership.element(), context, document),+ value(membership.array(), context, document));case Conjunction conjunction -> booleanNode(and(- truth(value(conjunction.left(), topic, document)),- truth(value(conjunction.right(), topic, document))));+ truth(value(conjunction.left(), context, document)),+ truth(value(conjunction.right(), context, document))));case Disjunction disjunction -> booleanNode(or(- truth(value(disjunction.left(), topic, document)),- truth(value(disjunction.right(), topic, document))));+ truth(value(disjunction.left(), context, document)),+ truth(value(disjunction.right(), context, document))));case Negation negation -> {- Boolean operand = truth(value(negation.operand(), topic, document));+ Boolean operand = truth(value(negation.operand(), context, document));yield operand == null ? null : booleanNode(!operand);}};@@ -145,17 +162,37 @@ public final class RuleSqlEvaluator {return current;}- private JsonNode call(Call call, String topic, ObjectNode document) {- if (call.function().equals("topic")) {- if (call.arguments().isEmpty()) {- return TextNode.valueOf(topic);+ private JsonNode call(Call call, RuleSqlContext context, ObjectNode document) {+ return switch (call.function()) {+ case "topic" -> topic(call, context);+ case "clientid" -> TextNode.valueOf(context.clientId() == null ? NO_MQTT_CLIENT : context.clientId());+ case "timestamp" -> LongNode.valueOf(clock.millis());+ case "accountid" -> TextNode.valueOf(context.accountId());+ case "newuuid" -> TextNode.valueOf(UUID.randomUUID().toString());+ case "isNull" -> {+ JsonNode argument = value(call.arguments().getFirst(), context, document);+ yield BooleanNode.valueOf(argument != null && argument.isNull());}- return call.arguments().getFirst() instanceof Literal position- ? topicSegment(topic, position.value().asLong())- : null;+ case "isUndefined" -> BooleanNode.valueOf(value(call.arguments().getFirst(), context, document) == null);+ default -> stringPredicate(call, context, document);+ };+ }++ private JsonNode topic(Call call, RuleSqlContext context) {+ if (call.arguments().isEmpty()) {+ return TextNode.valueOf(context.topic());}- String subject = text(value(call.arguments().get(0), topic, document));- String argument = text(value(call.arguments().get(1), topic, document));+ if (!(call.arguments().getFirst() instanceof Literal position)) {+ return null;+ }+ String[] segments = context.topic().split("/", -1);+ long index = position.value().asLong();+ return index < 1 || index > segments.length ? null : TextNode.valueOf(segments[(int) index - 1]);+ }++ private JsonNode stringPredicate(Call call, RuleSqlContext context, ObjectNode document) {+ String subject = text(value(call.arguments().get(0), context, document));+ String argument = text(value(call.arguments().get(1), context, document));if (subject == null || argument == null) {return null;}@@ -164,9 +201,17 @@ public final class RuleSqlEvaluator {: subject.endsWith(argument));}- private JsonNode topicSegment(String topic, long position) {- String[] segments = topic.split("/", -1);- return position < 1 || position > segments.length ? null : TextNode.valueOf(segments[(int) position - 1]);+ /** An array literal is undefined as a whole when any element is, so a position is never silently dropped. */+ private JsonNode array(ArrayLiteral literal, RuleSqlContext context, ObjectNode document) {+ ArrayNode array = objectMapper.createArrayNode();+ for (Expr element : literal.elements()) {+ JsonNode value = value(element, context, document);+ if (value == null) {+ return null;+ }+ array.add(value);+ }+ return array;}/**@@ -183,9 +228,9 @@ public final class RuleSqlEvaluator {return value.isContainerNode() ? value.toString() : value.asText();}- private JsonNode compare(Comparison comparison, String topic, ObjectNode document) {- JsonNode left = value(comparison.left(), topic, document);- JsonNode right = value(comparison.right(), topic, document);+ private JsonNode compare(Comparison comparison, RuleSqlContext context, ObjectNode document) {+ JsonNode left = value(comparison.left(), context, document);+ JsonNode right = value(comparison.right(), context, document);if (left == null || right == null) {return null;}@@ -200,14 +245,51 @@ public final class RuleSqlEvaluator {: booleanNode(matches(operator, leftNumber.compareTo(rightNumber)));}+ /** {@code IN} needs a defined element and an array; it is then a plain search with the equality rules. */+ private JsonNode contains(JsonNode element, JsonNode array) {+ if (element == null || array == null || !array.isArray()) {+ return null;+ }+ for (JsonNode candidate : array) {+ if (equal(element, candidate)) {+ return BooleanNode.TRUE;+ }+ }+ return BooleanNode.FALSE;+ }+/**- * AWS compares two numbers by value and everything else by type and value, so operands- * of different types are simply not equal, and {@code null} equals only {@code null}.+ * AWS compares two numbers by value, arrays element by element in order, objects by key+ * regardless of order, and everything else by type and value, so operands of different+ * types are simply not equal and {@code null} equals only {@code null}.*/private boolean equal(JsonNode left, JsonNode right) {if (left.isNumber() && right.isNumber()) {return left.decimalValue().compareTo(right.decimalValue()) == 0;}+ if (left.isArray() && right.isArray()) {+ if (left.size() != right.size()) {+ return false;+ }+ for (int i = 0; i < left.size(); i++) {+ if (!equal(left.get(i), right.get(i))) {+ return false;+ }+ }+ return true;+ }+ if (left.isObject() && right.isObject()) {+ if (left.size() != right.size()) {+ return false;+ }+ for (Map.Entry<String, JsonNode> property : left.properties()) {+ JsonNode other = right.get(property.getKey());+ if (other == null || !equal(property.getValue(), other)) {+ return false;+ }+ }+ return true;+ }return left.getNodeType() == right.getNodeType() && left.equals(right);}diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParser.java b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParser.javaindex 0d58252357..d28eaf874c 100644--- a/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParser.java+++ b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParser.java@@ -1,12 +1,14 @@package io.github.hectorvent.floci.services.iot.rules;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Aliased;+import io.github.hectorvent.floci.services.iot.rules.RuleSql.ArrayLiteral;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Call;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Comparison;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Conjunction;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Disjunction;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Expr;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Literal;+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Membership;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Negation;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Operator;import io.github.hectorvent.floci.services.iot.rules.RuleSql.Path;@@ -17,7 +19,11 @@ import java.math.BigDecimal;import java.util.ArrayList;import java.util.List;import java.util.Locale;+import java.util.Map;import java.util.Set;+import java.util.function.Function;+import java.util.stream.Collectors;+import java.util.stream.Stream;/*** Recursive descent parser for the subset of the AWS IoT rules SQL dialect Floci evaluates:@@ -27,10 +33,12 @@ import java.util.Set;* item := '*' | operand [AS identifier]* expr := term (OR term)** term := factor (AND factor)*- * factor := NOT factor | '(' expr ')' | operand [comparison operand]+ * factor := NOT factor | '(' expr ')' | operand [comparison operand | IN operand]* comparison := '=' | '<>' | '!=' | '<' | '<=' | '>' | '>='- * operand := path | literal | call+ * operand := path | literal | array | call+ * array := '[' [operand (',' operand)*] ']'* call := topic '(' [integer] ')' | (startswith | endswith) '(' operand ',' operand ')'+ * | (clientid | timestamp | accountid | newuuid) '(' ')' | (isNull | isUndefined) '(' operand ')'* path := identifier ('.' identifier)** literal := 'string' | "string" | number | TRUE | FALSE | NULL* </pre>@@ -41,7 +49,7 @@ import java.util.Set;public final class RuleSqlParser {private static final Set<String> KEYWORDS =- Set.of("SELECT", "FROM", "WHERE", "AS", "AND", "OR", "NOT", "TRUE", "FALSE", "NULL");+ Set.of("SELECT", "FROM", "WHERE", "AS", "AND", "OR", "NOT", "IN", "TRUE", "FALSE", "NULL");/*** Bounds the work one statement can cost, counting real tokens only: {@link #tokenize} appends@@ -51,8 +59,12 @@ public final class RuleSqlParser {*/private static final int MAX_TOKENS = 1000;- private static final String TOPIC = "topic";- private static final Set<String> STRING_PREDICATES = Set.of("startswith", "endswith");+ /** The supported functions by lower case name: the spelling AWS documents and the arguments each takes. */+ private static final Map<String, Signature> FUNCTIONS = Stream.of(+ new Signature("topic", 0, 1), new Signature("startswith", 2, 2), new Signature("endswith", 2, 2),+ new Signature("clientid", 0, 0), new Signature("timestamp", 0, 0), new Signature("accountid", 0, 0),+ new Signature("newuuid", 0, 0), new Signature("isNull", 1, 1), new Signature("isUndefined", 1, 1))+ .collect(Collectors.toMap(signature -> signature.name().toLowerCase(Locale.ROOT), Function.identity()));private final List<Token> tokens;private int index;@@ -146,6 +158,9 @@ public final class RuleSqlParser {return grouped;}Expr left = operand();+ if (matchKeyword("IN")) {+ return new Membership(left, operand());+ }Operator operator = comparisonOperator();return operator == null ? left : new Comparison(left, operator, operand());}@@ -197,8 +212,25 @@ public final class RuleSqlParser {advance();return peekIsSymbol("(") ? call(token) : path(token);}- default -> throw fail("Expected a field, literal or function", token);+ default -> {+ if (matchSymbol("[")) {+ return arrayLiteral();+ }+ throw fail("Expected a field, literal, array or function", token);+ }+ }+ }++ private Expr arrayLiteral() {+ List<Expr> elements = new ArrayList<>();+ if (!matchSymbol("]")) {+ elements.add(operand());+ while (matchSymbol(",")) {+ elements.add(operand());+ }+ expectSymbol("]");}+ return new ArrayLiteral(elements);}private Literal numberLiteral(Token token) {@@ -226,8 +258,8 @@ public final class RuleSqlParser {}private Expr call(Token name) {- String function = name.text().toLowerCase(Locale.ROOT);- if (!TOPIC.equals(function) && !STRING_PREDICATES.contains(function)) {+ Signature signature = FUNCTIONS.get(name.text().toLowerCase(Locale.ROOT));+ if (signature == null) {throw fail("Unsupported function", name);}expectSymbol("(");@@ -239,22 +271,17 @@ public final class RuleSqlParser {}}expectSymbol(")");- if (TOPIC.equals(function)) {- validateTopicArguments(arguments, name);- } else if (arguments.size() != 2) {- throw fail("Function " + name.text() + " takes two arguments", name);+ if (arguments.size() < signature.minArguments() || arguments.size() > signature.maxArguments()) {+ throw fail("Function " + signature.name() + " does not take " + arguments.size() + " arguments", name);+ }+ if (signature.name().equals("topic") && !arguments.isEmpty()) {+ validateTopicSegment(arguments.getFirst(), name);}- return new Call(function, arguments);+ return new Call(signature.name(), arguments);}- private void validateTopicArguments(List<Expr> arguments, Token name) {- if (arguments.isEmpty()) {- return;- }- if (arguments.size() != 1- || !(arguments.getFirst() instanceof Literal literal)- || !literal.value().isIntegralNumber()- || literal.value().asLong() < 1) {+ private void validateTopicSegment(Expr argument, Token name) {+ if (!(argument instanceof Literal literal) || !literal.value().isIntegralNumber() || literal.value().asLong() < 1) {throw fail("topic() takes no argument or a segment number starting at 1", name);}}@@ -388,7 +415,7 @@ public final class RuleSqlParser {return start + 2;}String one = sql.substring(start, start + 1);- if (!",.()*=<>".contains(one)) {+ if (!",.()[]*=<>".contains(one)) {throw new RuleSqlParseException("Unsupported character", one, start);}tokens.add(new Token(Kind.SYMBOL, one, start));@@ -401,4 +428,7 @@ public final class RuleSqlParser {private record Token(Kind kind, String text, int position) {}++ private record Signature(String name, int minArguments, int maxArguments) {+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.javaindex 297b0205b5..0ea12acd48 100644--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java@@ -354,6 +354,44 @@ class IotMqttEnabledIntegrationTest {}}+ @Test+ void clientidIsTheMqttClientThatPublishedTheMessage() throws Exception {+ given()+ .contentType("application/json")+ .body("""+ {+ "topicRulePayload": {+ "sql": "SELECT clientid() AS client, topic() AS topic FROM 'devices/phase8/mqtt/clientid'",+ "actions": [+ {+ "republish": {+ "roleArn": "arn:aws:iam::000000000000:role/iot-rule-role",+ "topic": "devices/phase8/mqtt/clientid-target"+ }+ }+ ]+ }+ }+ """)+ .when()+ .put("/rules/phase8MqttClientIdRule")+ .then()+ .statusCode(200);++ try (MqttTestClient subscriber = connectMqtt("phase8-clientid-sub")) {+ subscriber.subscribe("devices/phase8/mqtt/clientid-target");+ String publisherId = uniqueClientId("phase8-clientid-pub");++ try (MqttTestClient publisher = connectMqttClientId(publisherId)) {+ publisher.publish("devices/phase8/mqtt/clientid", "{}".getBytes(StandardCharsets.UTF_8));+ }++ MqttPublish republished = subscriber.takePublish();+ assertEquals("{\"client\":\"" + publisherId + "\",\"topic\":\"devices/phase8/mqtt/clientid\"}",+ new String(republished.payload(), StandardCharsets.UTF_8));+ }+ }+private MqttTestClient connectMqtt(String clientId) throws MqttException {return connectMqttClientId(uniqueClientId(clientId));}diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.javaindex 926a08ff1a..1faa712958 100644--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java@@ -138,7 +138,7 @@ class IotServiceTest {}private void publish(String payload) {- service.handlePublish(TOPIC, payload.getBytes(StandardCharsets.UTF_8), true, REGION);+ service.handlePublish(TOPIC, payload.getBytes(StandardCharsets.UTF_8), true, REGION, null);}private JsonNode capturedInvocationPayload(String functionArn) throws Exception {diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlIntegrationTest.javaindex 550f71bd95..803c87ad3a 100644--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlIntegrationTest.java@@ -14,6 +14,7 @@ import org.junit.jupiter.params.provider.CsvSource;import java.nio.charset.StandardCharsets;import java.util.List;+import static io.restassured.RestAssured.given;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertSame;@@ -52,7 +53,7 @@ class IotTopicRuleSqlIntegrationTest {@Testvoid aStatementOutsideTheSubsetKeepsFiringWithTheWholePayload() {- createRule("sqlPassthroughRule", "SELECT clientid() as client FROM 'sqltest/passthrough/+'",+ createRule("sqlPassthroughRule", "SELECT principal() as p FROM 'sqltest/passthrough/+'","sqltest/passthrough-out");publish("sqltest/passthrough/a", "{\"any\":1}");@@ -139,6 +140,42 @@ class IotTopicRuleSqlIntegrationTest {assertSame(parsed, compiledQuery("sqlToggleRule"));}+ @Test+ void messageFunctionsSeeTheRuleAccountAndNoMqttClientOnTheServicePath() {+ createRule("sqlFunctionsRule", "SELECT clientid() AS client, accountid() AS account, isUndefined(missing) AS u "+ + "FROM 'sqltest/functions/+' WHERE topic(2) IN ['functions'] AND isNull(n)", "sqltest/functions-out");++ publish("sqltest/functions/a", "{\"n\":null}");++ assertEquals(List.of("{\"client\":\"n/a\",\"account\":\"000000000000\",\"u\":true}"),+ republished("sqltest/functions-out"));+ }++ @Test+ void clientidIsTheMqttClientWhenThePublishCarriesOne() {+ createRule("sqlClientIdRule", "SELECT clientid() AS client FROM 'sqltest/clientid/+'", "sqltest/clientid-out");++ iotService.handlePublish("sqltest/clientid/a", "{}".getBytes(StandardCharsets.UTF_8), true, REGION, "sensor-9");++ assertEquals(List.of("{\"client\":\"sensor-9\"}"), republished("sqltest/clientid-out"));+ }++ @Test+ void anIotDataPublishOverHttpHasNoMqttClient() {+ createRule("sqlHttpClientIdRule", "SELECT clientid() AS client FROM 'sqltest/httpclient/+'",+ "sqltest/httpclient-out");++ given()+ .contentType("application/json")+ .body("{}")+ .when()+ .post("/topics/sqltest/httpclient/a")+ .then()+ .statusCode(200);++ assertEquals(List.of("{\"client\":\"n/a\"}"), republished("sqltest/httpclient-out"));+ }+@ParameterizedTest@CsvSource(delimiter = '|', value = {"SELECT *, topic() as topic FROM '$aws/things/+/shadow/name/building/update/accepted' WHERE endswith(clientToken, 'inbound') | $aws/things/sensor-1/shadow/name/building/update/accepted | {\"clientToken\":\"job:inbound\"} | true",@@ -188,7 +225,7 @@ class IotTopicRuleSqlIntegrationTest {}private void publish(String topic, String payload) {- iotService.handlePublish(topic, payload.getBytes(StandardCharsets.UTF_8), true, REGION);+ iotService.handlePublish(topic, payload.getBytes(StandardCharsets.UTF_8), true, REGION, null);}private List<String> republished(String targetTopic) {diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlStrictIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlStrictIntegrationTest.javaindex 0147b3a583..a1bd8f2607 100644--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlStrictIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlStrictIntegrationTest.java@@ -19,13 +19,13 @@ class IotTopicRuleSqlStrictIntegrationTest {void strictModeRejectsAStatementOutsideTheSubset() {given().contentType("application/json")- .body(rule("SELECT clientid() as client FROM 'strict/rules/+'"))+ .body(rule("SELECT principal() as p FROM 'strict/rules/+'")).when().put("/rules/strictRejectedRule").then().statusCode(400).body("__type", equalTo("SqlParseException"))- .body("message", containsString("clientid"));+ .body("message", containsString("principal"));}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlCorpusTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlCorpusTest.javaindex 45a85002ec..0e4e0a72ac 100644--- a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlCorpusTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlCorpusTest.java@@ -51,6 +51,16 @@ class RuleSqlCorpusTest {"SELECT * FROM 'topic/subtopic' WHERE startswith(topic(), 'topic') AND endswith(topic(2), 'topic')","SELECT * FROM 'a/b' WHERE temperature > -10 AND temperature < 100.5","SELECT * FROM 'a/b' WHERE name = 'o''brien'",+ "SELECT [lat,long] as lat_long FROM 'topic/subtopic'",+ "SELECT * FROM 'a/b' WHERE 3 IN arr",+ "SELECT clientid() AS client FROM 'a/b'",+ "SELECT *, timestamp() AS ts FROM 'a/b'",+ "SELECT *, accountid() AS account FROM 'a/b'",+ "SELECT newuuid() AS id FROM 'a/b'",+ "SELECT * FROM 'a/b' WHERE isUndefined(x)",+ "SELECT * FROM 'a/b' WHERE isNull(x)",+ "SELECT * FROM 'a/b' WHERE clientid() <> 'n/a' AND timestamp() > 0 AND accountid() = '123456789012'",+ "SELECT [lat, long] AS position, newuuid() AS id, isNull(x) AS n, isUndefined(y) AS u FROM 'a/b' WHERE x IN [1, 2] OR y IN arr","SELECT *\n FROM 'topic/subtopic'\n WHERE temperature > 50\n AND color <> 'red'"})void aStatementInsideTheSubsetParses(String sql) {@@ -64,17 +74,9 @@ class RuleSqlCorpusTest {"SELECT temperature * 1.8 + 32 AS f FROM 'a/b' | +","SELECT VALUE color FROM 'topic/subtopic' | color","SELECT {'latitude': get(lat_long, 0)} as lat_long FROM 'topic/subtopic' | {",- "SELECT [lat,long] as lat_long FROM 'topic/subtopic' | [",- "SELECT * FROM 'a/b' WHERE 3 IN arr | IN","SELECT * FROM 'a/b' WHERE exists (select * from arr as a where a = 3) | exists",- "SELECT clientid() AS client FROM 'a/b' | clientid",- "SELECT *, timestamp() AS ts FROM 'a/b' | timestamp",- "SELECT *, accountid() AS account FROM 'a/b' | accountid",- "SELECT newuuid() AS id FROM 'a/b' | newuuid","SELECT principal() AS p FROM 'a/b' | principal","SELECT * FROM 'a/b' WHERE traceid() <> '' | traceid",- "SELECT * FROM 'a/b' WHERE isUndefined(x) | isUndefined",- "SELECT * FROM 'a/b' WHERE isNull(x) | isNull","SELECT get(get(get(mydata,\"item2\"),\"0\"),\"my-key\") FROM 'iot/rules' | get","SELECT temp, md5(deviceid) AS hashed_id FROM 'topic/#' | md5","SELECT encode(*, 'base64') AS data FROM 'a/b' | encode",diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluatorTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluatorTest.javaindex c2286b45e9..046179f9b6 100644--- a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluatorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluatorTest.java@@ -7,6 +7,7 @@ import org.junit.jupiter.params.provider.CsvSource;import org.junit.jupiter.params.provider.ValueSource;import java.nio.charset.StandardCharsets;+import java.time.Clock;import java.util.Optional;import static org.junit.jupiter.api.Assertions.assertArrayEquals;@@ -18,7 +19,7 @@ class RuleSqlEvaluatorTest {private static final String SHADOW_TOPIC = "$aws/things/sensor-1/shadow/name/building/update/accepted";- private final RuleSqlEvaluator evaluator = new RuleSqlEvaluator(new ObjectMapper());+ private final RuleSqlEvaluator evaluator = new RuleSqlEvaluator(new ObjectMapper(), Clock.systemUTC());@Testvoid selectAllWithoutWhereForwardsTheOriginalBytes() {@@ -286,7 +287,7 @@ class RuleSqlEvaluatorTest {}private Optional<byte[]> evaluate(String sql, String topic, byte[] payload) {- return evaluator.evaluate("test-rule", RuleSqlParser.parse(sql), topic, payload);+ return evaluator.evaluate("test-rule", RuleSqlParser.parse(sql), new RuleSqlContext(topic, null, "000000000000"), payload);}private String text(Optional<byte[]> result) {diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlFunctionsTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlFunctionsTest.javanew file mode 100644index 0000000000..a91bbaf251--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlFunctionsTest.java@@ -0,0 +1,200 @@+package io.github.hectorvent.floci.services.iot.rules;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;++import java.nio.charset.StandardCharsets;+import java.time.Clock;+import java.time.Instant;+import java.time.ZoneOffset;+import java.util.Optional;+import java.util.UUID;++import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * The message functions, {@code isNull}, {@code isUndefined}, the {@code IN} operator and+ * array literals, against the tables in the AWS IoT SQL reference.+ */+class RuleSqlFunctionsTest {++ private static final long NOW = 1481825251155L;+ private static final Clock CLOCK = Clock.fixed(Instant.ofEpochMilli(NOW), ZoneOffset.UTC);+ private static final RuleSqlContext MQTT = new RuleSqlContext("a/b", "sensor-7", "123456789012");+ private static final RuleSqlContext HTTP = new RuleSqlContext("a/b", null, "123456789012");++ private final ObjectMapper objectMapper = new ObjectMapper();+ private final RuleSqlEvaluator evaluator = new RuleSqlEvaluator(objectMapper, CLOCK);++ @Test+ void timestampIsTheClockInMillisecondsSinceTheEpoch() {+ assertEquals("{\"ts\":" + NOW + "}", text(evaluate("SELECT timestamp() AS ts FROM 'a/b'", MQTT, "{}")));+ }++ @Test+ void timestampComparesAsANumber() {+ assertTrue(evaluate("SELECT * FROM 'a/b' WHERE timestamp() > 1481825251154", MQTT, "{}").isPresent());+ assertFalse(evaluate("SELECT * FROM 'a/b' WHERE timestamp() > 1481825251155", MQTT, "{}").isPresent());+ }++ @Test+ void newuuidIsAFreshUuidEachTime() throws Exception {+ String first = json(evaluate("SELECT newuuid() AS id FROM 'a/b'", MQTT, "{}")).get("id").textValue();+ String second = json(evaluate("SELECT newuuid() AS id FROM 'a/b'", MQTT, "{}")).get("id").textValue();++ assertDoesNotThrow(() -> UUID.fromString(first));+ assertDoesNotThrow(() -> UUID.fromString(second));+ assertNotEquals(first, second);+ }++ @Test+ void accountidIsTheAccountThatOwnsTheRule() {+ assertEquals("{\"account\":\"123456789012\"}",+ text(evaluate("SELECT accountid() AS account FROM 'a/b'", MQTT, "{}")));+ assertTrue(evaluate("SELECT * FROM 'a/b' WHERE accountid() = '123456789012'", HTTP, "{}").isPresent());+ }++ @Test+ void clientidIsTheMqttClientOrNaWhenTheMessageDidNotComeOverMqtt() {+ assertEquals("{\"client\":\"sensor-7\"}", text(evaluate("SELECT clientid() AS client FROM 'a/b'", MQTT, "{}")));+ assertEquals("{\"client\":\"n/a\"}", text(evaluate("SELECT clientid() AS client FROM 'a/b'", HTTP, "{}")));+ assertTrue(evaluate("SELECT * FROM 'a/b' WHERE startswith(clientid(), 'sensor-')", MQTT, "{}").isPresent());+ assertFalse(evaluate("SELECT * FROM 'a/b' WHERE startswith(clientid(), 'sensor-')", HTTP, "{}").isPresent());+ }++ @Test+ void functionsWithoutAnAliasAreWrittenUnderTheirOwnName() throws Exception {+ JsonNode document = json(evaluate(+ "SELECT timestamp(), accountid(), clientid(), isNull(n), isUndefined(missing) FROM 'a/b'",+ MQTT, "{\"n\":null}"));++ assertEquals(NOW, document.get("timestamp").longValue());+ assertEquals("123456789012", document.get("accountid").textValue());+ assertEquals("sensor-7", document.get("clientid").textValue());+ assertTrue(document.get("isNull").booleanValue());+ assertTrue(document.get("isUndefined").booleanValue());+ }++ @ParameterizedTest+ @CsvSource(delimiter = '|', value = {+ "i | false | false",+ "d | false | false",+ "b | false | false",+ "s | false | false",+ "a | false | false",+ "o | false | false",+ "n | true | false",+ "missing | false | true",+ "o.missing | false | true",+ "s.deeper | false | true",+ "NULL | true | false",+ "5 | false | false",+ "topic(9) | false | true",+ "topic(1) | false | false",+ "clientid() | false | false"+ })+ void isNullAndIsUndefinedFollowTheReferenceTables(String argument, boolean isNull, boolean isUndefined) {+ String payload = "{\"i\":1,\"d\":1.5,\"b\":true,\"s\":\"x\",\"a\":[1],\"o\":{},\"n\":null}";++ assertEquals("{\"n\":" + isNull + ",\"u\":" + isUndefined + "}", text(evaluate(+ "SELECT isNull(" + argument + ") AS n, isUndefined(" + argument + ") AS u FROM 'a/b'", MQTT, payload)));+ }++ @ParameterizedTest+ @CsvSource(delimiter = '|', quoteCharacter = '"', value = {+ "3 IN arr | true",+ "4 IN arr | false",+ "'three' IN arr | true",+ "5.7 IN arr | true",+ "3.0 IN arr | true",+ "NULL IN arr | true",+ "TRUE IN arr | false",+ "[1] IN arr | true",+ "num IN arr | true",+ "three IN arr | true",+ "s IN arr | false",+ "missing IN arr | false",+ "3 IN s | false",+ "3 IN missing | false",+ "3 IN o | false",+ "3 IN [1, 2, 3] | true",+ "'x' IN ['x', 'y'] | true",+ "'z' IN ['x', 'y'] | false",+ "3 IN [] | false",+ "s IN [s] | true",+ "NOT 3 IN [1] | true",+ "NOT missing IN [1] | false",+ "3 IN [1] OR num = 3 | true",+ "num IN arr AND s = 'x' | true",+ "topic(1) IN ['a', 'x'] | true",+ "'3' IN arr | false",+ "3 IN ['3'] | false"+ })+ void inChecksMembershipOfAnArrayWithTheEqualityRules(String predicate, boolean fires) {+ String payload = "{\"arr\":[1,2,3,\"three\",5.7,null,[1],{\"k\":1}],\"three\":\"three\",\"s\":\"x\",\"num\":3,\"o\":{}}";++ assertEquals(fires, evaluate("SELECT * FROM 'a/b' WHERE " + predicate, MQTT, payload).isPresent());+ }++ @Test+ void anArrayLiteralProjectsAsAnArray() {+ String payload = "{\"lat\":47.606,\"long\":-122.332}";++ assertEquals("{\"lat_long\":[47.606,-122.332]}",+ text(evaluate("SELECT [lat, long] AS lat_long FROM 'a/b'", MQTT, payload)));+ assertEquals("{\"mixed\":[1,\"a\",true,null,\"a\",[2],[]]}",+ text(evaluate("SELECT [1, 'a', TRUE, NULL, topic(1), [2], []] AS mixed FROM 'a/b'", MQTT, payload)));+ }++ @ParameterizedTest+ @CsvSource(delimiter = '|', value = {+ "1 IN list | true",+ "2 IN list | false",+ "NOT 2 IN list | true",+ "pair = [9007199254740993.0] | true",+ "pair = [9007199254740992] | false",+ "NOT pair = [9007199254740992] | true",+ "pair <> [9007199254740992] | true",+ "obj = obj | true",+ "NOT obj = obj | false",+ "list = [9007199254740993, 1.0] | true",+ "list = [9007199254740993, 1.00001] | false",+ "pair = [1, 2] | false",+ "pair = 'x' | false",+ "small = [0.30000000000000004] | true",+ "small = [0.3] | false"+ })+ void comparesNumbersInsideArraysAndObjectsExactly(String predicate, boolean fires) {+ String payload = "{\"list\":[9007199254740993.0, 1],\"pair\":[9007199254740993.0],"+ + "\"obj\":{\"k\":0.30000000000000004},\"small\":[0.30000000000000004]}";++ assertEquals(fires, evaluate("SELECT * FROM 'a/b' WHERE " + predicate, MQTT, payload).isPresent());+ }++ @Test+ void anArrayLiteralWithAnUndefinedElementIsUndefined() {+ assertEquals("{\"lat\":47.606}",+ text(evaluate("SELECT lat, [lat, missing] AS lat_long FROM 'a/b'", MQTT, "{\"lat\":47.606}")));+ assertFalse(evaluate("SELECT * FROM 'a/b' WHERE 1 IN [1, missing]", MQTT, "{}").isPresent());+ }++ private Optional<byte[]> evaluate(String sql, RuleSqlContext context, String payload) {+ return evaluator.evaluate("test-rule", RuleSqlParser.parse(sql), context, payload.getBytes(StandardCharsets.UTF_8));+ }++ private JsonNode json(Optional<byte[]> result) throws Exception {+ return objectMapper.readTree(text(result));+ }++ private String text(Optional<byte[]> result) {+ assertTrue(result.isPresent(), "Expected the rule to fire");+ return new String(result.get(), StandardCharsets.UTF_8);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlFuzzTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlFuzzTest.javaindex 988e4ef678..1af64b1836 100644--- a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlFuzzTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlFuzzTest.java@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;import org.junit.jupiter.api.Test;import java.nio.charset.StandardCharsets;+import java.time.Clock;import java.util.ArrayList;import java.util.Arrays;import java.util.List;@@ -35,14 +36,17 @@ class RuleSqlFuzzTest {"SELECT * FROM 'a/b' WHERE NOT active = FALSE AND (size > 10 OR weight <= -2.5) AND name <> NULL","select level, name from \"a/b\" where level != 3 and name = \"o''brien\"","SELECT NULL AS n, TRUE AS t, 'x' AS s, 1 AS i FROM 'topic/subtopic'",- "SELECT * FROM 'a/#' WHERE startswith(topic(2), 'b') AND endswith(topic(), 'c')");+ "SELECT * FROM 'a/#' WHERE startswith(topic(2), 'b') AND endswith(topic(), 'c')",+ "SELECT clientid() AS client, timestamp() AS ts, accountid() AS account, newuuid() AS id FROM 'a/b' WHERE clientid() <> 'n/a'",+ "SELECT [a, b, 1, 'x', NULL] AS list, isNull(n) AS n, isUndefined(missing) AS u FROM 'a/b' WHERE a IN arr OR 3 IN [1, 2, 3]");private static final String ALPHABET = " \t\n'\"(),.*=<>!-+[]{}$/#_abcXYZ019äé😀\\;:%";private static final List<String> TOKENS = List.of("SELECT", "FROM", "WHERE", "AS", "AND", "OR", "NOT", "TRUE", "FALSE", "NULL", "topic()", "topic(2)",- "topic(0)", "startswith(", "endswith(", "clientid()", "'x'", "\"y\"", "''", "42", "-1.5", "1e400",- "99999999999999999999", "*", ",", "(", ")", ".", "=", "<>", "!=", "<", "<=", ">", ">=", "a.b", "IN", "[", "]");+ "topic(0)", "startswith(", "endswith(", "clientid()", "timestamp()", "accountid()", "newuuid()", "isNull(",+ "isUndefined(", "principal()", "'x'", "\"y\"", "''", "42", "-1.5", "1e400", "99999999999999999999",+ "*", ",", "(", ")", ".", "=", "<>", "!=", "<", "<=", ">", ">=", "a.b", "IN", "[", "]", "[1, 'a']", "arr");private static final List<String> TOPICS = List.of("a/b/c", "$aws/things/x/shadow/update/accepted", "", "a");@@ -59,7 +63,7 @@ class RuleSqlFuzzTest {bytes(""),new byte[] {(byte) 0xff, (byte) 0xfe, 0x00, 0x7b});- private final RuleSqlEvaluator evaluator = new RuleSqlEvaluator(new ObjectMapper());+ private final RuleSqlEvaluator evaluator = new RuleSqlEvaluator(new ObjectMapper(), Clock.systemUTC());@Testvoid theParserOnlyEverThrowsRuleSqlParseExceptionAndTheEvaluatorNeverThrows() {@@ -109,7 +113,8 @@ class RuleSqlFuzzTest {}for (String topic : TOPICS) {for (byte[] payload : PAYLOADS) {- assertDoesNotThrow(() -> evaluator.evaluate("fuzz", query, topic, payload),+ assertDoesNotThrow(() -> evaluator.evaluate("fuzz", query,+ new RuleSqlContext(topic, topic.isEmpty() ? null : "fuzz-client", "000000000000"), payload),() -> "Evaluator threw for statement [" + sql + "] on topic [" + topic + "] with payload "+ new String(payload, StandardCharsets.UTF_8));}diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParserFunctionsTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParserFunctionsTest.javanew file mode 100644index 0000000000..d7280def74--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParserFunctionsTest.java@@ -0,0 +1,126 @@+package io.github.hectorvent.floci.services.iot.rules;++import io.github.hectorvent.floci.services.iot.rules.RuleSql.Aliased;+import io.github.hectorvent.floci.services.iot.rules.RuleSql.ArrayLiteral;+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Call;+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Comparison;+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Conjunction;+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Disjunction;+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Membership;+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Negation;+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Operator;+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Path;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;+import org.junit.jupiter.params.provider.ValueSource;++import java.util.List;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;++class RuleSqlParserFunctionsTest {++ @ParameterizedTest+ @CsvSource({+ "clientid, clientid",+ "CLIENTID, clientid",+ "timestamp, timestamp",+ "accountid, accountid",+ "newuuid, newuuid",+ "NewUUID, newuuid"+ })+ void parsesTheMessageFunctionsWithoutArgumentsUnderTheirCanonicalName(String written, String canonical) {+ RuleSql query = RuleSqlParser.parse("SELECT " + written + "() FROM 'a/b' WHERE " + written + "() = 'x'");++ assertEquals(List.of(new Aliased(new Call(canonical, List.of()), canonical)), query.projections());+ assertEquals(new Comparison(new Call(canonical, List.of()), Operator.EQUAL, RuleSql.text("x")), query.where());+ }++ @ParameterizedTest+ @CsvSource({+ "isNull, isNull",+ "isnull, isNull",+ "ISNULL, isNull",+ "isUndefined, isUndefined",+ "isundefined, isUndefined"+ })+ void parsesIsNullAndIsUndefinedWithOneArgumentUnderTheirCanonicalName(String written, String canonical) {+ RuleSql query = RuleSqlParser.parse("SELECT " + written + "(a.b) FROM 'a/b' WHERE " + written + "(topic(3))");++ assertEquals(List.of(new Aliased(new Call(canonical, List.of(new Path(List.of("a", "b")))), canonical)),+ query.projections());+ assertEquals(new Call(canonical, List.of(new Call("topic", List.of(RuleSql.number(3))))), query.where());+ }++ @Test+ void parsesInAgainstAFieldAndAgainstAnArrayLiteral() {+ assertEquals(new Membership(new Path(List.of("x")), new Path(List.of("arr"))),+ RuleSqlParser.parse("SELECT * FROM 't' WHERE x IN arr").where());+ assertEquals(new Membership(RuleSql.number(3), new ArrayLiteral(List.of(RuleSql.number(1), RuleSql.text("a")))),+ RuleSqlParser.parse("SELECT * FROM 't' WHERE 3 in [1, 'a']").where());+ }++ @Test+ void inBindsLikeAComparison() {+ Membership membership = new Membership(new Path(List.of("a")), new ArrayLiteral(List.of(RuleSql.number(1))));+ Comparison comparison = new Comparison(new Path(List.of("b")), Operator.EQUAL, RuleSql.number(2));++ assertEquals(new Conjunction(membership, comparison),+ RuleSqlParser.parse("SELECT * FROM 't' WHERE a IN [1] AND b = 2").where());+ assertEquals(new Disjunction(comparison, membership),+ RuleSqlParser.parse("SELECT * FROM 't' WHERE b = 2 OR a IN [1]").where());+ assertEquals(new Negation(membership),+ RuleSqlParser.parse("SELECT * FROM 't' WHERE NOT a IN [1]").where());+ }++ @Test+ void parsesArrayLiteralsOfAnyShapeAndRequiresAnAliasToSelectOne() {+ RuleSql query = RuleSqlParser.parse("SELECT [] AS empty, [1, 'a', TRUE, NULL, x.y, topic(1), [2]] AS mixed FROM 't'");++ assertEquals(List.of(+ new Aliased(new ArrayLiteral(List.of()), "empty"),+ new Aliased(new ArrayLiteral(List.of(+ RuleSql.number(1), RuleSql.text("a"), RuleSql.bool(true), RuleSql.nullValue(),+ new Path(List.of("x", "y")), new Call("topic", List.of(RuleSql.number(1))),+ new ArrayLiteral(List.of(RuleSql.number(2))))), "mixed")), query.projections());+ assertEquals("[", assertThrows(RuleSqlParseException.class,+ () -> RuleSqlParser.parse("SELECT [1] FROM 't'")).token());+ }++ @ParameterizedTest+ @ValueSource(strings = {"index", "inbound", "int_value", "IN_flag", "isNullable", "isUndefinedYet", "timestamps"})+ void aWordThatBeginsWithANewKeywordOrFunctionIsStillAFieldName(String field) {+ assertEquals(List.of(new Aliased(new Path(List.of(field)), field)),+ RuleSqlParser.parse("SELECT " + field + " FROM 't'").projections());+ }++ @ParameterizedTest+ @CsvSource(delimiter = '|', value = {+ "SELECT in FROM 't' | in",+ "SELECT * FROM 't' WHERE x IN | ''",+ "SELECT * FROM 't' WHERE IN [1] | IN",+ "SELECT * FROM 't' WHERE x IN [1, | ''",+ "SELECT * FROM 't' WHERE x IN [,] | ,",+ "SELECT * FROM 't' WHERE x IN [1 2] | 2",+ "SELECT * FROM 't' WHERE x IN [1]] | ]",+ "SELECT * FROM 't' WHERE x IN (1, 2) | (",+ "SELECT * FROM 't' WHERE x IN [1] IN [2] | IN",+ "SELECT * FROM 't' WHERE x = 1 IN [1] | IN",+ "SELECT * FROM 't' WHERE x NOT IN [1] | NOT",+ "SELECT timestamp(1) FROM 't' | timestamp",+ "SELECT clientid('x') FROM 't' | clientid",+ "SELECT accountid(a) FROM 't' | accountid",+ "SELECT newuuid(1) FROM 't' | newuuid",+ "SELECT isNull() AS n FROM 't' | isNull",+ "SELECT isNull(a, b) AS n FROM 't' | isNull",+ "SELECT isUndefined() AS u FROM 't' | isUndefined",+ "SELECT timestamp FROM 't' WHERE timestamp( | ''"+ })+ void rejectsMalformedUsesNamingTheToken(String sql, String token) {+ RuleSqlParseException failure = assertThrows(RuleSqlParseException.class, () -> RuleSqlParser.parse(sql));++ assertEquals(token, failure.token(), failure.getMessage());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParserTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParserTest.javaindex 027f69ee0a..3967567bca 100644--- a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParserTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParserTest.java@@ -143,16 +143,16 @@ class RuleSqlParserTest {@Testvoid reportsTheOffendingTokenAndPositionOfAnUnsupportedFunction() {RuleSqlParseException failure = assertThrows(RuleSqlParseException.class,- () -> RuleSqlParser.parse("SELECT clientid() FROM 'a/b'"));+ () -> RuleSqlParser.parse("SELECT principal() FROM 'a/b'"));- assertEquals("clientid", failure.token());+ assertEquals("principal", failure.token());assertEquals(7, failure.position());- assertTrue(failure.getMessage().contains("clientid"), failure.getMessage());+ assertTrue(failure.getMessage().contains("principal"), failure.getMessage());}@ParameterizedTest@CsvSource(delimiter = '|', value = {- "SELECT * FROM 'a/b' WHERE state IN ('on') | IN",+ "SELECT * FROM 'a/b' WHERE state IN ('on') | (","SELECT * FROM 'a/b' WHERE state IS NULL | IS","SELECT CASE WHEN a THEN 1 END FROM 'a/b' | WHEN","SELECT * FROM 'a/b' WHERE encode(a, 'base64') = 'x'| encode",