Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(iot): rule SQL message functions, isNull, isUndefined, IN and array literals (#3036)

Mario Toffiab88034cd1df6
diff --git a/docs/services/iot.md b/docs/services/iot.md
index c23065db1d..b4f29e1271 100644
--- a/docs/services/iot.md
+++ b/docs/services/iot.md
@@ -219,10 +219,12 @@ statement := SELECT item (',' item)* FROM '<topic filter>' [WHERE expr]
item := '*' | operand [AS identifier]
expr := term (OR term)*
term := factor (AND factor)*
-factor := NOT factor | '(' expr ')' | operand [comparison operand]
+factor := NOT factor | '(' expr ')' | operand [comparison operand | IN operand]
comparison := '=' | '<>' | '!=' | '<' | '<=' | '>' | '>='
-operand := path | literal | call
+operand := path | literal | array | call
+array := '[' [operand (',' operand)*] ']'
call := topic() | topic(<segment>) | startswith(operand, operand) | endswith(operand, operand)
+ | clientid() | timestamp() | accountid() | newuuid() | isNull(operand) | isUndefined(operand)
path := identifier ('.' identifier)*
literal := 'string' | "string" | number | TRUE | FALSE | NULL
```
@@ -231,6 +233,16 @@ Semantics:
- Keywords and function names are case insensitive, field names are case sensitive.
- `topic()` is the full MQTT topic, `topic(n)` is its nth segment counting from 1.
+- `clientid()` is the MQTT client that published the message, or `n/a` for an IoT Data `Publish`
+ over HTTP, as on AWS. `accountid()` is the account that owns the rule. `timestamp()` is the
+ current time in milliseconds since the epoch. `newuuid()` is a fresh random UUID.
+- `isNull(x)` is true only for a JSON `null`, `isUndefined(x)` only for a missing field or an
+ undefined expression. Neither is ever undefined itself.
+- `x IN y` is true when the array `y` holds a value equal to `x` under the equality rules below,
+ false otherwise. It is undefined when `x` is undefined or `y` is not an array. `y` may be a
+ payload field or an array literal such as `[1, 'a']`.
+- An array literal is undefined as a whole when any of its elements is, so a position is never
+ silently dropped.
- A select item without `AS` is written under the last segment of its path, or under the function
name, so `topic()` becomes `topic`.
- When `*` is present, every payload field is copied first and the other select items are written
@@ -244,9 +256,10 @@ Semantics:
`clientToken <> 'x'`.
- JSON `null` is a value, not `Undefined`: it equals only `NULL`, so `clientToken <> 'x'` is true
when the field is null and undefined when it is missing.
-- `=` and `<>` compare two numbers by value and anything else by type and value, so operands of
- different types are simply not equal: `level = '3'` is false and `level <> '3'` is true when
- `level` is the number 3. These follow the operator tables in the AWS IoT SQL reference.
+- `=` and `<>` compare two numbers by value, arrays element by element in order, objects by key in
+ any order, and anything else by type and value, so operands of different types are simply not
+ equal: `level = '3'` is false and `level <> '3'` is true when `level` is the number 3. These
+ follow the operator tables in the AWS IoT SQL reference.
- `<`, `<=`, `>` and `>=` convert both operands to a number. A string converts when it looks like
one (`'10' > 9` is true); any other operand makes the comparison undefined.
- Payload numbers are read exactly, never through a double, so `9007199254740993.0`, `1e-400` and
@@ -269,10 +282,11 @@ statement with `SqlParseException` instead, the way AWS does. It is `false` by d
Current limitations:
-- Not evaluated: `clientid()`, `timestamp()`, `accountid()`, `principal()`, `newuuid()`, `isNull()`,
- `isUndefined()`, `encode()`, `get_thing_shadow()`, arithmetic, array indexing, `IN`, `CASE`, and
- `${}` substitution templates in action fields. A rule using any of them takes the unparsed path
- described above.
+- Not evaluated: `principal()`, which needs the certificate a device authenticated with and the
+ broker does not verify one yet; `traceid()`; `encode()`, `get()`, `get_thing_shadow()` and the
+ rest of the AWS function list; arithmetic; array indexing; `CASE`; `SELECT VALUE`; object
+ literals; the `SET` clause; `EXISTS` subqueries; and `${}` substitution templates in action
+ fields. A rule using any of them takes the unparsed path described above.
- `awsIotSqlVersion` is stored and echoed back but not acted on. The versions differ in how
`SELECT *` treats arrays, which Floci does not model.
- Less common AWS IoT rule action types are follow-up scope.
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotDataController.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotDataController.java
index 7613b26e43..c347acc3dd 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotDataController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotDataController.java
@@ -91,7 +91,7 @@ public class IotDataController {
byte[] payload) {
// Unsigned or non-SigV4 publishes carry no usable region; both count as unresolved
String region = regionResolver.resolveRegionFromAuthOrNull(headers.getHeaderString("Authorization"));
- iotService.publish(topic, payload == null ? new byte[0] : payload, Boolean.TRUE.equals(retain), qos == null ? 0 : qos, region);
+ iotService.publish(topic, payload == null ? new byte[0] : payload, Boolean.TRUE.equals(retain), qos == null ? 0 : qos, region, null);
return Response.ok(objectMapper.createObjectNode()).build();
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
index 37c05b7878..52c167fcc8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotMqttBrokerService.java
@@ -446,7 +446,7 @@ public class IotMqttBrokerService {
return;
}
- iotService.get().publish(topic, payload, message.isRetain(), message.qosLevel().value(), null);
+ iotService.get().publish(topic, payload, message.isRetain(), message.qosLevel().value(), null, session.clientId());
fanOut(topic, payload, false);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java
index 442cf11482..9e3581c22c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotService.java
@@ -30,6 +30,7 @@ import io.github.hectorvent.floci.services.iot.model.IotTopicRule;
import io.github.hectorvent.floci.services.acm.CertificateGenerator;
import io.github.hectorvent.floci.services.iot.model.Thing;
import io.github.hectorvent.floci.services.iot.rules.RuleSql;
+import io.github.hectorvent.floci.services.iot.rules.RuleSqlContext;
import io.github.hectorvent.floci.services.iot.rules.RuleSqlEvaluator;
import io.github.hectorvent.floci.services.iot.rules.RuleSqlParseException;
import io.github.hectorvent.floci.services.iot.rules.RuleSqlParser;
@@ -52,6 +53,7 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateEncodingException;
import java.security.cert.X509Certificate;
+import java.time.Clock;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
@@ -214,7 +216,7 @@ public class IotService {
this.cloudWatchLogsService = cloudWatchLogsService;
this.certificateAuthority = certificateAuthority;
this.policyEvaluator = policyEvaluator;
- this.ruleSqlEvaluator = new RuleSqlEvaluator(objectMapper);
+ this.ruleSqlEvaluator = new RuleSqlEvaluator(objectMapper, Clock.systemUTC());
}
public String describeEndpoint(String endpointType) {
@@ -876,10 +878,11 @@ public class IotService {
}
public void publish(String topic, byte[] payload) {
- publish(topic, payload, false, 0, null);
+ publish(topic, payload, false, 0, null, null);
}
- public void publish(String topic, byte[] payload, boolean retain, int qos, String region) {
+ /** {@code clientId} is the MQTT client that published, or null for a message that did not come over MQTT. */
+ public void publish(String topic, byte[] payload, boolean retain, int qos, String region, String clientId) {
byte[] eventPayload = payload == null ? new byte[0] : payload;
if (retain) {
if (eventPayload.length == 0) {
@@ -893,7 +896,7 @@ public class IotService {
retainedMessageStore.put(retainedMessageKey(topic), retained);
}
}
- handlePublish(topic, eventPayload, true, region);
+ handlePublish(topic, eventPayload, true, region, clientId);
}
public void deleteConnection(String clientId, boolean cleanSession) {
@@ -1277,7 +1280,7 @@ public class IotService {
topicRuleStore.put(topicRuleKey(region, ruleName), rule);
}
- void handlePublish(String topic, byte[] payload, boolean evaluateRules, String region) {
+ void handlePublish(String topic, byte[] payload, boolean evaluateRules, String region, String clientId) {
byte[] eventPayload = payload == null ? new byte[0] : payload;
publishEventRecorder.record(topic, eventPayload);
if (!evaluateRules) {
@@ -1285,7 +1288,7 @@ public class IotService {
}
for (IotTopicRule rule : rulesForPublish(region)) {
if (!rule.isRuleDisabled()) {
- matchAndProject(rule, topic, eventPayload)
+ matchAndProject(rule, topic, clientId, eventPayload)
.ifPresent(document -> executeTopicRule(rule, topic, eventPayload, document));
}
}
@@ -1296,7 +1299,7 @@ public class IotService {
* fire for this message. Rules whose SQL could not be parsed take the pre-parser path: the
* topic filter is read straight out of the SQL and the payload is forwarded untouched.
*/
- private Optional<byte[]> matchAndProject(IotTopicRule rule, String topic, byte[] payload) {
+ private Optional<byte[]> matchAndProject(IotTopicRule rule, String topic, String clientId, byte[] payload) {
RuleSql query = ruleQuery(rule);
if (query == null) {
return topicMatches(extractTopicPattern(rule.getSql()), topic) ? Optional.of(payload) : Optional.empty();
@@ -1304,7 +1307,9 @@ public class IotService {
if (!topicMatches(query.topicFilter(), topic)) {
return Optional.empty();
}
- return ruleSqlEvaluator.evaluate(rule.getRuleName(), query, topic, payload);
+ RuleSqlContext context = new RuleSqlContext(topic, clientId,
+ AwsArnUtils.accountOrDefault(rule.getRuleArn(), config.defaultAccountId()));
+ return ruleSqlEvaluator.evaluate(rule.getRuleName(), query, context, payload);
}
/**
@@ -1486,7 +1491,7 @@ public class IotService {
case "republish" -> {
String targetTopic = action.path("topic").asText(null);
if (targetTopic != null && !targetTopic.isBlank()) {
- handlePublish(targetTopic, payload, false, region);
+ handlePublish(targetTopic, payload, false, region, null);
mqttBrokerService.publish(targetTopic, payload);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSql.java b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSql.java
index 8d09b1b7a5..0b426a3d78 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSql.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSql.java
@@ -56,7 +56,8 @@ public record RuleSql(List<Projection> projections, String topicFilter, Expr whe
public record Aliased(Expr expression, String alias) implements Projection {
}
- public sealed interface Expr permits Path, Literal, Call, Comparison, Conjunction, Disjunction, Negation {
+ public sealed interface Expr
+ permits Path, Literal, Call, ArrayLiteral, Comparison, Membership, Conjunction, Disjunction, Negation {
}
/** A dotted field path such as {@code state.reported.temperature}. */
@@ -69,16 +70,27 @@ public record RuleSql(List<Projection> projections, String topicFilter, Expr whe
public record Literal(JsonNode value) implements Expr {
}
- /** A supported function call: {@code topic}, {@code startswith} or {@code endswith}. */
+ /** A supported function call under the name AWS documents, such as {@code topic} or {@code isNull}. */
public record Call(String function, List<Expr> arguments) implements Expr {
public Call {
arguments = List.copyOf(arguments);
}
}
+ /** A JSON array written in the statement, such as {@code [lat, long]}. */
+ public record ArrayLiteral(List<Expr> elements) implements Expr {
+ public ArrayLiteral {
+ elements = List.copyOf(elements);
+ }
+ }
+
public record Comparison(Expr left, Operator operator, Expr right) implements Expr {
}
+ /** {@code element IN array}: true when the array holds a value equal to the element. */
+ public record Membership(Expr element, Expr array) implements Expr {
+ }
+
public record Conjunction(Expr left, Expr right) implements Expr {
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlContext.java b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlContext.java
new file mode 100644
index 0000000000..a137162aa9
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlContext.java
@@ -0,0 +1,9 @@
+package io.github.hectorvent.floci.services.iot.rules;
+
+/**
+ * What a rule can ask about the message beyond its payload: the topic it was published on,
+ * the MQTT client that sent it ({@code null} when it did not arrive over MQTT, which
+ * {@code clientid()} reports as {@code n/a} as AWS does), and the account that owns the rule.
+ */
+public record RuleSqlContext(String topic, String clientId, String accountId) {
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluator.java b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluator.java
index 4189295ad8..7e6fc5e33c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluator.java
@@ -4,17 +4,21 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.BooleanNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.LongNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Aliased;
+import io.github.hectorvent.floci.services.iot.rules.RuleSql.ArrayLiteral;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Call;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Comparison;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Conjunction;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Disjunction;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Expr;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Literal;
+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Membership;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Negation;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Operator;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Path;
@@ -25,18 +29,21 @@ import org.jboss.logging.Logger;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
+import java.time.Clock;
+import java.util.Map;
import java.util.Optional;
+import java.util.UUID;
import java.util.regex.Pattern;
/**
* Evaluates a parsed topic rule statement against one published message, following the
- * operator and conversion tables of the AWS IoT SQL reference.
+ * operator, function and conversion tables of the AWS IoT SQL reference.
*
* <p>A value AWS reports as {@code Undefined} (a missing field, an out of range topic segment,
* a function argument that cannot be converted) is a {@code null} {@link JsonNode}. It spreads:
- * a comparison, {@code AND}, {@code OR} or {@code NOT} with an undefined operand is undefined,
- * and a rule fires only when its {@code WHERE} is true. JSON {@code null} is not undefined: it
- * is a value that equals only itself.
+ * a comparison, {@code IN}, {@code AND}, {@code OR} or {@code NOT} with an undefined operand is
+ * undefined, and a rule fires only when its {@code WHERE} is true. JSON {@code null} is not
+ * undefined: it is a value that equals only itself.
*
* <p>Payload numbers are read as {@code BigDecimal}, never through a {@code double}, so they
* compare exactly at any size or precision, as AWS's Decimal does.
@@ -48,34 +55,40 @@ public final class RuleSqlEvaluator {
/** The string forms AWS converts to a number when an ordering operator meets a string. */
private static final Pattern NUMERIC_STRING = Pattern.compile("-?\\d+(\\.\\d+)?([eE]-?\\d+)?");
+ /** What {@code clientid()} returns for a message that did not arrive over MQTT, as AWS documents. */
+ private static final String NO_MQTT_CLIENT = "n/a";
+
private final ObjectMapper objectMapper;
private final ObjectReader payloadReader;
+ private final Clock clock;
- public RuleSqlEvaluator(ObjectMapper objectMapper) {
+ public RuleSqlEvaluator(ObjectMapper objectMapper, Clock clock) {
this.objectMapper = objectMapper;
this.payloadReader = objectMapper.reader()
.with(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS)
.with(JsonNodeFactory.withExactBigDecimals(true));
+ this.clock = clock;
}
/**
* Returns the document the rule's actions should receive, or empty when the rule does
* not fire for this message.
*/
- public Optional<byte[]> evaluate(String ruleName, RuleSql query, String topic, byte[] payload) {
+ public Optional<byte[]> evaluate(String ruleName, RuleSql query, RuleSqlContext context, byte[] payload) {
byte[] message = payload == null ? new byte[0] : payload;
if (query.isSelectAllOnly() && query.where() == null) {
return Optional.of(message);
}
ObjectNode document = readObject(message);
if (document == null) {
- LOG.debugv("Topic rule {0} skipped a payload that is not a JSON object on topic {1}", ruleName, topic);
+ LOG.debugv("Topic rule {0} skipped a payload that is not a JSON object on topic {1}",
+ ruleName, context.topic());
return Optional.empty();
}
- if (query.where() != null && !Boolean.TRUE.equals(truth(value(query.where(), topic, document)))) {
+ if (query.where() != null && !Boolean.TRUE.equals(truth(value(query.where(), context, document)))) {
return Optional.empty();
}
- return Optional.of(query.isSelectAllOnly() ? message : project(query, topic, document));
+ return Optional.of(query.isSelectAllOnly() ? message : project(query, context, document));
}
private ObjectNode readObject(byte[] payload) {
@@ -96,14 +109,14 @@ public final class RuleSqlEvaluator {
* present, so a select item whose alias collides with a payload field overwrites it,
* which is what AWS does for statements such as {@code SELECT *, topic() as topic}.
*/
- private byte[] project(RuleSql query, String topic, ObjectNode document) {
+ private byte[] project(RuleSql query, RuleSqlContext context, ObjectNode document) {
ObjectNode projected = objectMapper.createObjectNode();
if (query.projections().stream().anyMatch(SelectAll.class::isInstance)) {
projected.setAll(document);
}
for (Projection projection : query.projections()) {
if (projection instanceof Aliased aliased) {
- JsonNode value = value(aliased.expression(), topic, document);
+ JsonNode value = value(aliased.expression(), context, document);
if (value != null) {
projected.set(aliased.alias(), value);
}
@@ -112,20 +125,24 @@ public final class RuleSqlEvaluator {
return projected.toString().getBytes(StandardCharsets.UTF_8);
}
- private JsonNode value(Expr expression, String topic, ObjectNode document) {
+ private JsonNode value(Expr expression, RuleSqlContext context, ObjectNode document) {
return switch (expression) {
case Literal literal -> literal.value();
case Path path -> resolve(path, document);
- case Call call -> call(call, topic, document);
- case Comparison comparison -> compare(comparison, topic, document);
+ case Call call -> call(call, context, document);
+ case ArrayLiteral array -> array(array, context, document);
+ case Comparison comparison -> compare(comparison, context, document);
+ case Membership membership -> contains(
+ value(membership.element(), context, document),
+ value(membership.array(), context, document));
case Conjunction conjunction -> booleanNode(and(
- truth(value(conjunction.left(), topic, document)),
- truth(value(conjunction.right(), topic, document))));
+ truth(value(conjunction.left(), context, document)),
+ truth(value(conjunction.right(), context, document))));
case Disjunction disjunction -> booleanNode(or(
- truth(value(disjunction.left(), topic, document)),
- truth(value(disjunction.right(), topic, document))));
+ truth(value(disjunction.left(), context, document)),
+ truth(value(disjunction.right(), context, document))));
case Negation negation -> {
- Boolean operand = truth(value(negation.operand(), topic, document));
+ Boolean operand = truth(value(negation.operand(), context, document));
yield operand == null ? null : booleanNode(!operand);
}
};
@@ -145,17 +162,37 @@ public final class RuleSqlEvaluator {
return current;
}
- private JsonNode call(Call call, String topic, ObjectNode document) {
- if (call.function().equals("topic")) {
- if (call.arguments().isEmpty()) {
- return TextNode.valueOf(topic);
+ private JsonNode call(Call call, RuleSqlContext context, ObjectNode document) {
+ return switch (call.function()) {
+ case "topic" -> topic(call, context);
+ case "clientid" -> TextNode.valueOf(context.clientId() == null ? NO_MQTT_CLIENT : context.clientId());
+ case "timestamp" -> LongNode.valueOf(clock.millis());
+ case "accountid" -> TextNode.valueOf(context.accountId());
+ case "newuuid" -> TextNode.valueOf(UUID.randomUUID().toString());
+ case "isNull" -> {
+ JsonNode argument = value(call.arguments().getFirst(), context, document);
+ yield BooleanNode.valueOf(argument != null && argument.isNull());
}
- return call.arguments().getFirst() instanceof Literal position
- ? topicSegment(topic, position.value().asLong())
- : null;
+ case "isUndefined" -> BooleanNode.valueOf(value(call.arguments().getFirst(), context, document) == null);
+ default -> stringPredicate(call, context, document);
+ };
+ }
+
+ private JsonNode topic(Call call, RuleSqlContext context) {
+ if (call.arguments().isEmpty()) {
+ return TextNode.valueOf(context.topic());
}
- String subject = text(value(call.arguments().get(0), topic, document));
- String argument = text(value(call.arguments().get(1), topic, document));
+ if (!(call.arguments().getFirst() instanceof Literal position)) {
+ return null;
+ }
+ String[] segments = context.topic().split("/", -1);
+ long index = position.value().asLong();
+ return index < 1 || index > segments.length ? null : TextNode.valueOf(segments[(int) index - 1]);
+ }
+
+ private JsonNode stringPredicate(Call call, RuleSqlContext context, ObjectNode document) {
+ String subject = text(value(call.arguments().get(0), context, document));
+ String argument = text(value(call.arguments().get(1), context, document));
if (subject == null || argument == null) {
return null;
}
@@ -164,9 +201,17 @@ public final class RuleSqlEvaluator {
: subject.endsWith(argument));
}
- private JsonNode topicSegment(String topic, long position) {
- String[] segments = topic.split("/", -1);
- return position < 1 || position > segments.length ? null : TextNode.valueOf(segments[(int) position - 1]);
+ /** An array literal is undefined as a whole when any element is, so a position is never silently dropped. */
+ private JsonNode array(ArrayLiteral literal, RuleSqlContext context, ObjectNode document) {
+ ArrayNode array = objectMapper.createArrayNode();
+ for (Expr element : literal.elements()) {
+ JsonNode value = value(element, context, document);
+ if (value == null) {
+ return null;
+ }
+ array.add(value);
+ }
+ return array;
}
/**
@@ -183,9 +228,9 @@ public final class RuleSqlEvaluator {
return value.isContainerNode() ? value.toString() : value.asText();
}
- private JsonNode compare(Comparison comparison, String topic, ObjectNode document) {
- JsonNode left = value(comparison.left(), topic, document);
- JsonNode right = value(comparison.right(), topic, document);
+ private JsonNode compare(Comparison comparison, RuleSqlContext context, ObjectNode document) {
+ JsonNode left = value(comparison.left(), context, document);
+ JsonNode right = value(comparison.right(), context, document);
if (left == null || right == null) {
return null;
}
@@ -200,14 +245,51 @@ public final class RuleSqlEvaluator {
: booleanNode(matches(operator, leftNumber.compareTo(rightNumber)));
}
+ /** {@code IN} needs a defined element and an array; it is then a plain search with the equality rules. */
+ private JsonNode contains(JsonNode element, JsonNode array) {
+ if (element == null || array == null || !array.isArray()) {
+ return null;
+ }
+ for (JsonNode candidate : array) {
+ if (equal(element, candidate)) {
+ return BooleanNode.TRUE;
+ }
+ }
+ return BooleanNode.FALSE;
+ }
+
/**
- * AWS compares two numbers by value and everything else by type and value, so operands
- * of different types are simply not equal, and {@code null} equals only {@code null}.
+ * AWS compares two numbers by value, arrays element by element in order, objects by key
+ * regardless of order, and everything else by type and value, so operands of different
+ * types are simply not equal and {@code null} equals only {@code null}.
*/
private boolean equal(JsonNode left, JsonNode right) {
if (left.isNumber() && right.isNumber()) {
return left.decimalValue().compareTo(right.decimalValue()) == 0;
}
+ if (left.isArray() && right.isArray()) {
+ if (left.size() != right.size()) {
+ return false;
+ }
+ for (int i = 0; i < left.size(); i++) {
+ if (!equal(left.get(i), right.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+ if (left.isObject() && right.isObject()) {
+ if (left.size() != right.size()) {
+ return false;
+ }
+ for (Map.Entry<String, JsonNode> property : left.properties()) {
+ JsonNode other = right.get(property.getKey());
+ if (other == null || !equal(property.getValue(), other)) {
+ return false;
+ }
+ }
+ return true;
+ }
return left.getNodeType() == right.getNodeType() && left.equals(right);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParser.java b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParser.java
index 0d58252357..d28eaf874c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParser.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParser.java
@@ -1,12 +1,14 @@
package io.github.hectorvent.floci.services.iot.rules;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Aliased;
+import io.github.hectorvent.floci.services.iot.rules.RuleSql.ArrayLiteral;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Call;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Comparison;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Conjunction;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Disjunction;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Expr;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Literal;
+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Membership;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Negation;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Operator;
import io.github.hectorvent.floci.services.iot.rules.RuleSql.Path;
@@ -17,7 +19,11 @@ import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
/**
* Recursive descent parser for the subset of the AWS IoT rules SQL dialect Floci evaluates:
@@ -27,10 +33,12 @@ import java.util.Set;
* item := '*' | operand [AS identifier]
* expr := term (OR term)*
* term := factor (AND factor)*
- * factor := NOT factor | '(' expr ')' | operand [comparison operand]
+ * factor := NOT factor | '(' expr ')' | operand [comparison operand | IN operand]
* comparison := '=' | '<>' | '!=' | '<' | '<=' | '>' | '>='
- * operand := path | literal | call
+ * operand := path | literal | array | call
+ * array := '[' [operand (',' operand)*] ']'
* call := topic '(' [integer] ')' | (startswith | endswith) '(' operand ',' operand ')'
+ * | (clientid | timestamp | accountid | newuuid) '(' ')' | (isNull | isUndefined) '(' operand ')'
* path := identifier ('.' identifier)*
* literal := 'string' | "string" | number | TRUE | FALSE | NULL
* </pre>
@@ -41,7 +49,7 @@ import java.util.Set;
public final class RuleSqlParser {
private static final Set<String> KEYWORDS =
- Set.of("SELECT", "FROM", "WHERE", "AS", "AND", "OR", "NOT", "TRUE", "FALSE", "NULL");
+ Set.of("SELECT", "FROM", "WHERE", "AS", "AND", "OR", "NOT", "IN", "TRUE", "FALSE", "NULL");
/**
* Bounds the work one statement can cost, counting real tokens only: {@link #tokenize} appends
@@ -51,8 +59,12 @@ public final class RuleSqlParser {
*/
private static final int MAX_TOKENS = 1000;
- private static final String TOPIC = "topic";
- private static final Set<String> STRING_PREDICATES = Set.of("startswith", "endswith");
+ /** The supported functions by lower case name: the spelling AWS documents and the arguments each takes. */
+ private static final Map<String, Signature> FUNCTIONS = Stream.of(
+ new Signature("topic", 0, 1), new Signature("startswith", 2, 2), new Signature("endswith", 2, 2),
+ new Signature("clientid", 0, 0), new Signature("timestamp", 0, 0), new Signature("accountid", 0, 0),
+ new Signature("newuuid", 0, 0), new Signature("isNull", 1, 1), new Signature("isUndefined", 1, 1))
+ .collect(Collectors.toMap(signature -> signature.name().toLowerCase(Locale.ROOT), Function.identity()));
private final List<Token> tokens;
private int index;
@@ -146,6 +158,9 @@ public final class RuleSqlParser {
return grouped;
}
Expr left = operand();
+ if (matchKeyword("IN")) {
+ return new Membership(left, operand());
+ }
Operator operator = comparisonOperator();
return operator == null ? left : new Comparison(left, operator, operand());
}
@@ -197,8 +212,25 @@ public final class RuleSqlParser {
advance();
return peekIsSymbol("(") ? call(token) : path(token);
}
- default -> throw fail("Expected a field, literal or function", token);
+ default -> {
+ if (matchSymbol("[")) {
+ return arrayLiteral();
+ }
+ throw fail("Expected a field, literal, array or function", token);
+ }
+ }
+ }
+
+ private Expr arrayLiteral() {
+ List<Expr> elements = new ArrayList<>();
+ if (!matchSymbol("]")) {
+ elements.add(operand());
+ while (matchSymbol(",")) {
+ elements.add(operand());
+ }
+ expectSymbol("]");
}
+ return new ArrayLiteral(elements);
}
private Literal numberLiteral(Token token) {
@@ -226,8 +258,8 @@ public final class RuleSqlParser {
}
private Expr call(Token name) {
- String function = name.text().toLowerCase(Locale.ROOT);
- if (!TOPIC.equals(function) && !STRING_PREDICATES.contains(function)) {
+ Signature signature = FUNCTIONS.get(name.text().toLowerCase(Locale.ROOT));
+ if (signature == null) {
throw fail("Unsupported function", name);
}
expectSymbol("(");
@@ -239,22 +271,17 @@ public final class RuleSqlParser {
}
}
expectSymbol(")");
- if (TOPIC.equals(function)) {
- validateTopicArguments(arguments, name);
- } else if (arguments.size() != 2) {
- throw fail("Function " + name.text() + " takes two arguments", name);
+ if (arguments.size() < signature.minArguments() || arguments.size() > signature.maxArguments()) {
+ throw fail("Function " + signature.name() + " does not take " + arguments.size() + " arguments", name);
+ }
+ if (signature.name().equals("topic") && !arguments.isEmpty()) {
+ validateTopicSegment(arguments.getFirst(), name);
}
- return new Call(function, arguments);
+ return new Call(signature.name(), arguments);
}
- private void validateTopicArguments(List<Expr> arguments, Token name) {
- if (arguments.isEmpty()) {
- return;
- }
- if (arguments.size() != 1
- || !(arguments.getFirst() instanceof Literal literal)
- || !literal.value().isIntegralNumber()
- || literal.value().asLong() < 1) {
+ private void validateTopicSegment(Expr argument, Token name) {
+ if (!(argument instanceof Literal literal) || !literal.value().isIntegralNumber() || literal.value().asLong() < 1) {
throw fail("topic() takes no argument or a segment number starting at 1", name);
}
}
@@ -388,7 +415,7 @@ public final class RuleSqlParser {
return start + 2;
}
String one = sql.substring(start, start + 1);
- if (!",.()*=<>".contains(one)) {
+ if (!",.()[]*=<>".contains(one)) {
throw new RuleSqlParseException("Unsupported character", one, start);
}
tokens.add(new Token(Kind.SYMBOL, one, start));
@@ -401,4 +428,7 @@ public final class RuleSqlParser {
private record Token(Kind kind, String text, int position) {
}
+
+ private record Signature(String name, int minArguments, int maxArguments) {
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java
index 297b0205b5..0ea12acd48 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotMqttEnabledIntegrationTest.java
@@ -354,6 +354,44 @@ class IotMqttEnabledIntegrationTest {
}
}
+ @Test
+ void clientidIsTheMqttClientThatPublishedTheMessage() throws Exception {
+ given()
+ .contentType("application/json")
+ .body("""
+ {
+ "topicRulePayload": {
+ "sql": "SELECT clientid() AS client, topic() AS topic FROM 'devices/phase8/mqtt/clientid'",
+ "actions": [
+ {
+ "republish": {
+ "roleArn": "arn:aws:iam::000000000000:role/iot-rule-role",
+ "topic": "devices/phase8/mqtt/clientid-target"
+ }
+ }
+ ]
+ }
+ }
+ """)
+ .when()
+ .put("/rules/phase8MqttClientIdRule")
+ .then()
+ .statusCode(200);
+
+ try (MqttTestClient subscriber = connectMqtt("phase8-clientid-sub")) {
+ subscriber.subscribe("devices/phase8/mqtt/clientid-target");
+ String publisherId = uniqueClientId("phase8-clientid-pub");
+
+ try (MqttTestClient publisher = connectMqttClientId(publisherId)) {
+ publisher.publish("devices/phase8/mqtt/clientid", "{}".getBytes(StandardCharsets.UTF_8));
+ }
+
+ MqttPublish republished = subscriber.takePublish();
+ assertEquals("{\"client\":\"" + publisherId + "\",\"topic\":\"devices/phase8/mqtt/clientid\"}",
+ new String(republished.payload(), StandardCharsets.UTF_8));
+ }
+ }
+
private MqttTestClient connectMqtt(String clientId) throws MqttException {
return connectMqttClientId(uniqueClientId(clientId));
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java
index 926a08ff1a..1faa712958 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotServiceTest.java
@@ -138,7 +138,7 @@ class IotServiceTest {
}
private void publish(String payload) {
- service.handlePublish(TOPIC, payload.getBytes(StandardCharsets.UTF_8), true, REGION);
+ service.handlePublish(TOPIC, payload.getBytes(StandardCharsets.UTF_8), true, REGION, null);
}
private JsonNode capturedInvocationPayload(String functionArn) throws Exception {
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlIntegrationTest.java
index 550f71bd95..803c87ad3a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlIntegrationTest.java
@@ -14,6 +14,7 @@ import org.junit.jupiter.params.provider.CsvSource;
import java.nio.charset.StandardCharsets;
import java.util.List;
+import static io.restassured.RestAssured.given;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertSame;
@@ -52,7 +53,7 @@ class IotTopicRuleSqlIntegrationTest {
@Test
void aStatementOutsideTheSubsetKeepsFiringWithTheWholePayload() {
- createRule("sqlPassthroughRule", "SELECT clientid() as client FROM 'sqltest/passthrough/+'",
+ createRule("sqlPassthroughRule", "SELECT principal() as p FROM 'sqltest/passthrough/+'",
"sqltest/passthrough-out");
publish("sqltest/passthrough/a", "{\"any\":1}");
@@ -139,6 +140,42 @@ class IotTopicRuleSqlIntegrationTest {
assertSame(parsed, compiledQuery("sqlToggleRule"));
}
+ @Test
+ void messageFunctionsSeeTheRuleAccountAndNoMqttClientOnTheServicePath() {
+ createRule("sqlFunctionsRule", "SELECT clientid() AS client, accountid() AS account, isUndefined(missing) AS u "
+ + "FROM 'sqltest/functions/+' WHERE topic(2) IN ['functions'] AND isNull(n)", "sqltest/functions-out");
+
+ publish("sqltest/functions/a", "{\"n\":null}");
+
+ assertEquals(List.of("{\"client\":\"n/a\",\"account\":\"000000000000\",\"u\":true}"),
+ republished("sqltest/functions-out"));
+ }
+
+ @Test
+ void clientidIsTheMqttClientWhenThePublishCarriesOne() {
+ createRule("sqlClientIdRule", "SELECT clientid() AS client FROM 'sqltest/clientid/+'", "sqltest/clientid-out");
+
+ iotService.handlePublish("sqltest/clientid/a", "{}".getBytes(StandardCharsets.UTF_8), true, REGION, "sensor-9");
+
+ assertEquals(List.of("{\"client\":\"sensor-9\"}"), republished("sqltest/clientid-out"));
+ }
+
+ @Test
+ void anIotDataPublishOverHttpHasNoMqttClient() {
+ createRule("sqlHttpClientIdRule", "SELECT clientid() AS client FROM 'sqltest/httpclient/+'",
+ "sqltest/httpclient-out");
+
+ given()
+ .contentType("application/json")
+ .body("{}")
+ .when()
+ .post("/topics/sqltest/httpclient/a")
+ .then()
+ .statusCode(200);
+
+ assertEquals(List.of("{\"client\":\"n/a\"}"), republished("sqltest/httpclient-out"));
+ }
+
@ParameterizedTest
@CsvSource(delimiter = '|', value = {
"SELECT *, topic() as topic FROM '$aws/things/+/shadow/name/building/update/accepted' WHERE endswith(clientToken, 'inbound') | $aws/things/sensor-1/shadow/name/building/update/accepted | {\"clientToken\":\"job:inbound\"} | true",
@@ -188,7 +225,7 @@ class IotTopicRuleSqlIntegrationTest {
}
private void publish(String topic, String payload) {
- iotService.handlePublish(topic, payload.getBytes(StandardCharsets.UTF_8), true, REGION);
+ iotService.handlePublish(topic, payload.getBytes(StandardCharsets.UTF_8), true, REGION, null);
}
private List<String> republished(String targetTopic) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlStrictIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlStrictIntegrationTest.java
index 0147b3a583..a1bd8f2607 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlStrictIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotTopicRuleSqlStrictIntegrationTest.java
@@ -19,13 +19,13 @@ class IotTopicRuleSqlStrictIntegrationTest {
void strictModeRejectsAStatementOutsideTheSubset() {
given()
.contentType("application/json")
- .body(rule("SELECT clientid() as client FROM 'strict/rules/+'"))
+ .body(rule("SELECT principal() as p FROM 'strict/rules/+'"))
.when()
.put("/rules/strictRejectedRule")
.then()
.statusCode(400)
.body("__type", equalTo("SqlParseException"))
- .body("message", containsString("clientid"));
+ .body("message", containsString("principal"));
}
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlCorpusTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlCorpusTest.java
index 45a85002ec..0e4e0a72ac 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlCorpusTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlCorpusTest.java
@@ -51,6 +51,16 @@ class RuleSqlCorpusTest {
"SELECT * FROM 'topic/subtopic' WHERE startswith(topic(), 'topic') AND endswith(topic(2), 'topic')",
"SELECT * FROM 'a/b' WHERE temperature > -10 AND temperature < 100.5",
"SELECT * FROM 'a/b' WHERE name = 'o''brien'",
+ "SELECT [lat,long] as lat_long FROM 'topic/subtopic'",
+ "SELECT * FROM 'a/b' WHERE 3 IN arr",
+ "SELECT clientid() AS client FROM 'a/b'",
+ "SELECT *, timestamp() AS ts FROM 'a/b'",
+ "SELECT *, accountid() AS account FROM 'a/b'",
+ "SELECT newuuid() AS id FROM 'a/b'",
+ "SELECT * FROM 'a/b' WHERE isUndefined(x)",
+ "SELECT * FROM 'a/b' WHERE isNull(x)",
+ "SELECT * FROM 'a/b' WHERE clientid() <> 'n/a' AND timestamp() > 0 AND accountid() = '123456789012'",
+ "SELECT [lat, long] AS position, newuuid() AS id, isNull(x) AS n, isUndefined(y) AS u FROM 'a/b' WHERE x IN [1, 2] OR y IN arr",
"SELECT *\n FROM 'topic/subtopic'\n WHERE temperature > 50\n AND color <> 'red'"
})
void aStatementInsideTheSubsetParses(String sql) {
@@ -64,17 +74,9 @@ class RuleSqlCorpusTest {
"SELECT temperature * 1.8 + 32 AS f FROM 'a/b' | +",
"SELECT VALUE color FROM 'topic/subtopic' | color",
"SELECT {'latitude': get(lat_long, 0)} as lat_long FROM 'topic/subtopic' | {",
- "SELECT [lat,long] as lat_long FROM 'topic/subtopic' | [",
- "SELECT * FROM 'a/b' WHERE 3 IN arr | IN",
"SELECT * FROM 'a/b' WHERE exists (select * from arr as a where a = 3) | exists",
- "SELECT clientid() AS client FROM 'a/b' | clientid",
- "SELECT *, timestamp() AS ts FROM 'a/b' | timestamp",
- "SELECT *, accountid() AS account FROM 'a/b' | accountid",
- "SELECT newuuid() AS id FROM 'a/b' | newuuid",
"SELECT principal() AS p FROM 'a/b' | principal",
"SELECT * FROM 'a/b' WHERE traceid() <> '' | traceid",
- "SELECT * FROM 'a/b' WHERE isUndefined(x) | isUndefined",
- "SELECT * FROM 'a/b' WHERE isNull(x) | isNull",
"SELECT get(get(get(mydata,\"item2\"),\"0\"),\"my-key\") FROM 'iot/rules' | get",
"SELECT temp, md5(deviceid) AS hashed_id FROM 'topic/#' | md5",
"SELECT encode(*, 'base64') AS data FROM 'a/b' | encode",
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluatorTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluatorTest.java
index c2286b45e9..046179f9b6 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluatorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlEvaluatorTest.java
@@ -7,6 +7,7 @@ import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
import java.nio.charset.StandardCharsets;
+import java.time.Clock;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
@@ -18,7 +19,7 @@ class RuleSqlEvaluatorTest {
private static final String SHADOW_TOPIC = "$aws/things/sensor-1/shadow/name/building/update/accepted";
- private final RuleSqlEvaluator evaluator = new RuleSqlEvaluator(new ObjectMapper());
+ private final RuleSqlEvaluator evaluator = new RuleSqlEvaluator(new ObjectMapper(), Clock.systemUTC());
@Test
void selectAllWithoutWhereForwardsTheOriginalBytes() {
@@ -286,7 +287,7 @@ class RuleSqlEvaluatorTest {
}
private Optional<byte[]> evaluate(String sql, String topic, byte[] payload) {
- return evaluator.evaluate("test-rule", RuleSqlParser.parse(sql), topic, payload);
+ return evaluator.evaluate("test-rule", RuleSqlParser.parse(sql), new RuleSqlContext(topic, null, "000000000000"), payload);
}
private String text(Optional<byte[]> result) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlFunctionsTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlFunctionsTest.java
new file mode 100644
index 0000000000..a91bbaf251
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlFunctionsTest.java
@@ -0,0 +1,200 @@
+package io.github.hectorvent.floci.services.iot.rules;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.Optional;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * The message functions, {@code isNull}, {@code isUndefined}, the {@code IN} operator and
+ * array literals, against the tables in the AWS IoT SQL reference.
+ */
+class RuleSqlFunctionsTest {
+
+ private static final long NOW = 1481825251155L;
+ private static final Clock CLOCK = Clock.fixed(Instant.ofEpochMilli(NOW), ZoneOffset.UTC);
+ private static final RuleSqlContext MQTT = new RuleSqlContext("a/b", "sensor-7", "123456789012");
+ private static final RuleSqlContext HTTP = new RuleSqlContext("a/b", null, "123456789012");
+
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private final RuleSqlEvaluator evaluator = new RuleSqlEvaluator(objectMapper, CLOCK);
+
+ @Test
+ void timestampIsTheClockInMillisecondsSinceTheEpoch() {
+ assertEquals("{\"ts\":" + NOW + "}", text(evaluate("SELECT timestamp() AS ts FROM 'a/b'", MQTT, "{}")));
+ }
+
+ @Test
+ void timestampComparesAsANumber() {
+ assertTrue(evaluate("SELECT * FROM 'a/b' WHERE timestamp() > 1481825251154", MQTT, "{}").isPresent());
+ assertFalse(evaluate("SELECT * FROM 'a/b' WHERE timestamp() > 1481825251155", MQTT, "{}").isPresent());
+ }
+
+ @Test
+ void newuuidIsAFreshUuidEachTime() throws Exception {
+ String first = json(evaluate("SELECT newuuid() AS id FROM 'a/b'", MQTT, "{}")).get("id").textValue();
+ String second = json(evaluate("SELECT newuuid() AS id FROM 'a/b'", MQTT, "{}")).get("id").textValue();
+
+ assertDoesNotThrow(() -> UUID.fromString(first));
+ assertDoesNotThrow(() -> UUID.fromString(second));
+ assertNotEquals(first, second);
+ }
+
+ @Test
+ void accountidIsTheAccountThatOwnsTheRule() {
+ assertEquals("{\"account\":\"123456789012\"}",
+ text(evaluate("SELECT accountid() AS account FROM 'a/b'", MQTT, "{}")));
+ assertTrue(evaluate("SELECT * FROM 'a/b' WHERE accountid() = '123456789012'", HTTP, "{}").isPresent());
+ }
+
+ @Test
+ void clientidIsTheMqttClientOrNaWhenTheMessageDidNotComeOverMqtt() {
+ assertEquals("{\"client\":\"sensor-7\"}", text(evaluate("SELECT clientid() AS client FROM 'a/b'", MQTT, "{}")));
+ assertEquals("{\"client\":\"n/a\"}", text(evaluate("SELECT clientid() AS client FROM 'a/b'", HTTP, "{}")));
+ assertTrue(evaluate("SELECT * FROM 'a/b' WHERE startswith(clientid(), 'sensor-')", MQTT, "{}").isPresent());
+ assertFalse(evaluate("SELECT * FROM 'a/b' WHERE startswith(clientid(), 'sensor-')", HTTP, "{}").isPresent());
+ }
+
+ @Test
+ void functionsWithoutAnAliasAreWrittenUnderTheirOwnName() throws Exception {
+ JsonNode document = json(evaluate(
+ "SELECT timestamp(), accountid(), clientid(), isNull(n), isUndefined(missing) FROM 'a/b'",
+ MQTT, "{\"n\":null}"));
+
+ assertEquals(NOW, document.get("timestamp").longValue());
+ assertEquals("123456789012", document.get("accountid").textValue());
+ assertEquals("sensor-7", document.get("clientid").textValue());
+ assertTrue(document.get("isNull").booleanValue());
+ assertTrue(document.get("isUndefined").booleanValue());
+ }
+
+ @ParameterizedTest
+ @CsvSource(delimiter = '|', value = {
+ "i | false | false",
+ "d | false | false",
+ "b | false | false",
+ "s | false | false",
+ "a | false | false",
+ "o | false | false",
+ "n | true | false",
+ "missing | false | true",
+ "o.missing | false | true",
+ "s.deeper | false | true",
+ "NULL | true | false",
+ "5 | false | false",
+ "topic(9) | false | true",
+ "topic(1) | false | false",
+ "clientid() | false | false"
+ })
+ void isNullAndIsUndefinedFollowTheReferenceTables(String argument, boolean isNull, boolean isUndefined) {
+ String payload = "{\"i\":1,\"d\":1.5,\"b\":true,\"s\":\"x\",\"a\":[1],\"o\":{},\"n\":null}";
+
+ assertEquals("{\"n\":" + isNull + ",\"u\":" + isUndefined + "}", text(evaluate(
+ "SELECT isNull(" + argument + ") AS n, isUndefined(" + argument + ") AS u FROM 'a/b'", MQTT, payload)));
+ }
+
+ @ParameterizedTest
+ @CsvSource(delimiter = '|', quoteCharacter = '"', value = {
+ "3 IN arr | true",
+ "4 IN arr | false",
+ "'three' IN arr | true",
+ "5.7 IN arr | true",
+ "3.0 IN arr | true",
+ "NULL IN arr | true",
+ "TRUE IN arr | false",
+ "[1] IN arr | true",
+ "num IN arr | true",
+ "three IN arr | true",
+ "s IN arr | false",
+ "missing IN arr | false",
+ "3 IN s | false",
+ "3 IN missing | false",
+ "3 IN o | false",
+ "3 IN [1, 2, 3] | true",
+ "'x' IN ['x', 'y'] | true",
+ "'z' IN ['x', 'y'] | false",
+ "3 IN [] | false",
+ "s IN [s] | true",
+ "NOT 3 IN [1] | true",
+ "NOT missing IN [1] | false",
+ "3 IN [1] OR num = 3 | true",
+ "num IN arr AND s = 'x' | true",
+ "topic(1) IN ['a', 'x'] | true",
+ "'3' IN arr | false",
+ "3 IN ['3'] | false"
+ })
+ void inChecksMembershipOfAnArrayWithTheEqualityRules(String predicate, boolean fires) {
+ String payload = "{\"arr\":[1,2,3,\"three\",5.7,null,[1],{\"k\":1}],\"three\":\"three\",\"s\":\"x\",\"num\":3,\"o\":{}}";
+
+ assertEquals(fires, evaluate("SELECT * FROM 'a/b' WHERE " + predicate, MQTT, payload).isPresent());
+ }
+
+ @Test
+ void anArrayLiteralProjectsAsAnArray() {
+ String payload = "{\"lat\":47.606,\"long\":-122.332}";
+
+ assertEquals("{\"lat_long\":[47.606,-122.332]}",
+ text(evaluate("SELECT [lat, long] AS lat_long FROM 'a/b'", MQTT, payload)));
+ assertEquals("{\"mixed\":[1,\"a\",true,null,\"a\",[2],[]]}",
+ text(evaluate("SELECT [1, 'a', TRUE, NULL, topic(1), [2], []] AS mixed FROM 'a/b'", MQTT, payload)));
+ }
+
+ @ParameterizedTest
+ @CsvSource(delimiter = '|', value = {
+ "1 IN list | true",
+ "2 IN list | false",
+ "NOT 2 IN list | true",
+ "pair = [9007199254740993.0] | true",
+ "pair = [9007199254740992] | false",
+ "NOT pair = [9007199254740992] | true",
+ "pair <> [9007199254740992] | true",
+ "obj = obj | true",
+ "NOT obj = obj | false",
+ "list = [9007199254740993, 1.0] | true",
+ "list = [9007199254740993, 1.00001] | false",
+ "pair = [1, 2] | false",
+ "pair = 'x' | false",
+ "small = [0.30000000000000004] | true",
+ "small = [0.3] | false"
+ })
+ void comparesNumbersInsideArraysAndObjectsExactly(String predicate, boolean fires) {
+ String payload = "{\"list\":[9007199254740993.0, 1],\"pair\":[9007199254740993.0],"
+ + "\"obj\":{\"k\":0.30000000000000004},\"small\":[0.30000000000000004]}";
+
+ assertEquals(fires, evaluate("SELECT * FROM 'a/b' WHERE " + predicate, MQTT, payload).isPresent());
+ }
+
+ @Test
+ void anArrayLiteralWithAnUndefinedElementIsUndefined() {
+ assertEquals("{\"lat\":47.606}",
+ text(evaluate("SELECT lat, [lat, missing] AS lat_long FROM 'a/b'", MQTT, "{\"lat\":47.606}")));
+ assertFalse(evaluate("SELECT * FROM 'a/b' WHERE 1 IN [1, missing]", MQTT, "{}").isPresent());
+ }
+
+ private Optional<byte[]> evaluate(String sql, RuleSqlContext context, String payload) {
+ return evaluator.evaluate("test-rule", RuleSqlParser.parse(sql), context, payload.getBytes(StandardCharsets.UTF_8));
+ }
+
+ private JsonNode json(Optional<byte[]> result) throws Exception {
+ return objectMapper.readTree(text(result));
+ }
+
+ private String text(Optional<byte[]> result) {
+ assertTrue(result.isPresent(), "Expected the rule to fire");
+ return new String(result.get(), StandardCharsets.UTF_8);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlFuzzTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlFuzzTest.java
index 988e4ef678..1af64b1836 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlFuzzTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlFuzzTest.java
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import java.nio.charset.StandardCharsets;
+import java.time.Clock;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -35,14 +36,17 @@ class RuleSqlFuzzTest {
"SELECT * FROM 'a/b' WHERE NOT active = FALSE AND (size > 10 OR weight <= -2.5) AND name <> NULL",
"select level, name from \"a/b\" where level != 3 and name = \"o''brien\"",
"SELECT NULL AS n, TRUE AS t, 'x' AS s, 1 AS i FROM 'topic/subtopic'",
- "SELECT * FROM 'a/#' WHERE startswith(topic(2), 'b') AND endswith(topic(), 'c')");
+ "SELECT * FROM 'a/#' WHERE startswith(topic(2), 'b') AND endswith(topic(), 'c')",
+ "SELECT clientid() AS client, timestamp() AS ts, accountid() AS account, newuuid() AS id FROM 'a/b' WHERE clientid() <> 'n/a'",
+ "SELECT [a, b, 1, 'x', NULL] AS list, isNull(n) AS n, isUndefined(missing) AS u FROM 'a/b' WHERE a IN arr OR 3 IN [1, 2, 3]");
private static final String ALPHABET = " \t\n'\"(),.*=<>!-+[]{}$/#_abcXYZ019äé😀\\;:%";
private static final List<String> TOKENS = List.of(
"SELECT", "FROM", "WHERE", "AS", "AND", "OR", "NOT", "TRUE", "FALSE", "NULL", "topic()", "topic(2)",
- "topic(0)", "startswith(", "endswith(", "clientid()", "'x'", "\"y\"", "''", "42", "-1.5", "1e400",
- "99999999999999999999", "*", ",", "(", ")", ".", "=", "<>", "!=", "<", "<=", ">", ">=", "a.b", "IN", "[", "]");
+ "topic(0)", "startswith(", "endswith(", "clientid()", "timestamp()", "accountid()", "newuuid()", "isNull(",
+ "isUndefined(", "principal()", "'x'", "\"y\"", "''", "42", "-1.5", "1e400", "99999999999999999999",
+ "*", ",", "(", ")", ".", "=", "<>", "!=", "<", "<=", ">", ">=", "a.b", "IN", "[", "]", "[1, 'a']", "arr");
private static final List<String> TOPICS = List.of("a/b/c", "$aws/things/x/shadow/update/accepted", "", "a");
@@ -59,7 +63,7 @@ class RuleSqlFuzzTest {
bytes(""),
new byte[] {(byte) 0xff, (byte) 0xfe, 0x00, 0x7b});
- private final RuleSqlEvaluator evaluator = new RuleSqlEvaluator(new ObjectMapper());
+ private final RuleSqlEvaluator evaluator = new RuleSqlEvaluator(new ObjectMapper(), Clock.systemUTC());
@Test
void theParserOnlyEverThrowsRuleSqlParseExceptionAndTheEvaluatorNeverThrows() {
@@ -109,7 +113,8 @@ class RuleSqlFuzzTest {
}
for (String topic : TOPICS) {
for (byte[] payload : PAYLOADS) {
- assertDoesNotThrow(() -> evaluator.evaluate("fuzz", query, topic, payload),
+ assertDoesNotThrow(() -> evaluator.evaluate("fuzz", query,
+ new RuleSqlContext(topic, topic.isEmpty() ? null : "fuzz-client", "000000000000"), payload),
() -> "Evaluator threw for statement [" + sql + "] on topic [" + topic + "] with payload "
+ new String(payload, StandardCharsets.UTF_8));
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParserFunctionsTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParserFunctionsTest.java
new file mode 100644
index 0000000000..d7280def74
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParserFunctionsTest.java
@@ -0,0 +1,126 @@
+package io.github.hectorvent.floci.services.iot.rules;
+
+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Aliased;
+import io.github.hectorvent.floci.services.iot.rules.RuleSql.ArrayLiteral;
+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Call;
+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Comparison;
+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Conjunction;
+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Disjunction;
+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Membership;
+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Negation;
+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Operator;
+import io.github.hectorvent.floci.services.iot.rules.RuleSql.Path;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class RuleSqlParserFunctionsTest {
+
+ @ParameterizedTest
+ @CsvSource({
+ "clientid, clientid",
+ "CLIENTID, clientid",
+ "timestamp, timestamp",
+ "accountid, accountid",
+ "newuuid, newuuid",
+ "NewUUID, newuuid"
+ })
+ void parsesTheMessageFunctionsWithoutArgumentsUnderTheirCanonicalName(String written, String canonical) {
+ RuleSql query = RuleSqlParser.parse("SELECT " + written + "() FROM 'a/b' WHERE " + written + "() = 'x'");
+
+ assertEquals(List.of(new Aliased(new Call(canonical, List.of()), canonical)), query.projections());
+ assertEquals(new Comparison(new Call(canonical, List.of()), Operator.EQUAL, RuleSql.text("x")), query.where());
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "isNull, isNull",
+ "isnull, isNull",
+ "ISNULL, isNull",
+ "isUndefined, isUndefined",
+ "isundefined, isUndefined"
+ })
+ void parsesIsNullAndIsUndefinedWithOneArgumentUnderTheirCanonicalName(String written, String canonical) {
+ RuleSql query = RuleSqlParser.parse("SELECT " + written + "(a.b) FROM 'a/b' WHERE " + written + "(topic(3))");
+
+ assertEquals(List.of(new Aliased(new Call(canonical, List.of(new Path(List.of("a", "b")))), canonical)),
+ query.projections());
+ assertEquals(new Call(canonical, List.of(new Call("topic", List.of(RuleSql.number(3))))), query.where());
+ }
+
+ @Test
+ void parsesInAgainstAFieldAndAgainstAnArrayLiteral() {
+ assertEquals(new Membership(new Path(List.of("x")), new Path(List.of("arr"))),
+ RuleSqlParser.parse("SELECT * FROM 't' WHERE x IN arr").where());
+ assertEquals(new Membership(RuleSql.number(3), new ArrayLiteral(List.of(RuleSql.number(1), RuleSql.text("a")))),
+ RuleSqlParser.parse("SELECT * FROM 't' WHERE 3 in [1, 'a']").where());
+ }
+
+ @Test
+ void inBindsLikeAComparison() {
+ Membership membership = new Membership(new Path(List.of("a")), new ArrayLiteral(List.of(RuleSql.number(1))));
+ Comparison comparison = new Comparison(new Path(List.of("b")), Operator.EQUAL, RuleSql.number(2));
+
+ assertEquals(new Conjunction(membership, comparison),
+ RuleSqlParser.parse("SELECT * FROM 't' WHERE a IN [1] AND b = 2").where());
+ assertEquals(new Disjunction(comparison, membership),
+ RuleSqlParser.parse("SELECT * FROM 't' WHERE b = 2 OR a IN [1]").where());
+ assertEquals(new Negation(membership),
+ RuleSqlParser.parse("SELECT * FROM 't' WHERE NOT a IN [1]").where());
+ }
+
+ @Test
+ void parsesArrayLiteralsOfAnyShapeAndRequiresAnAliasToSelectOne() {
+ RuleSql query = RuleSqlParser.parse("SELECT [] AS empty, [1, 'a', TRUE, NULL, x.y, topic(1), [2]] AS mixed FROM 't'");
+
+ assertEquals(List.of(
+ new Aliased(new ArrayLiteral(List.of()), "empty"),
+ new Aliased(new ArrayLiteral(List.of(
+ RuleSql.number(1), RuleSql.text("a"), RuleSql.bool(true), RuleSql.nullValue(),
+ new Path(List.of("x", "y")), new Call("topic", List.of(RuleSql.number(1))),
+ new ArrayLiteral(List.of(RuleSql.number(2))))), "mixed")), query.projections());
+ assertEquals("[", assertThrows(RuleSqlParseException.class,
+ () -> RuleSqlParser.parse("SELECT [1] FROM 't'")).token());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"index", "inbound", "int_value", "IN_flag", "isNullable", "isUndefinedYet", "timestamps"})
+ void aWordThatBeginsWithANewKeywordOrFunctionIsStillAFieldName(String field) {
+ assertEquals(List.of(new Aliased(new Path(List.of(field)), field)),
+ RuleSqlParser.parse("SELECT " + field + " FROM 't'").projections());
+ }
+
+ @ParameterizedTest
+ @CsvSource(delimiter = '|', value = {
+ "SELECT in FROM 't' | in",
+ "SELECT * FROM 't' WHERE x IN | ''",
+ "SELECT * FROM 't' WHERE IN [1] | IN",
+ "SELECT * FROM 't' WHERE x IN [1, | ''",
+ "SELECT * FROM 't' WHERE x IN [,] | ,",
+ "SELECT * FROM 't' WHERE x IN [1 2] | 2",
+ "SELECT * FROM 't' WHERE x IN [1]] | ]",
+ "SELECT * FROM 't' WHERE x IN (1, 2) | (",
+ "SELECT * FROM 't' WHERE x IN [1] IN [2] | IN",
+ "SELECT * FROM 't' WHERE x = 1 IN [1] | IN",
+ "SELECT * FROM 't' WHERE x NOT IN [1] | NOT",
+ "SELECT timestamp(1) FROM 't' | timestamp",
+ "SELECT clientid('x') FROM 't' | clientid",
+ "SELECT accountid(a) FROM 't' | accountid",
+ "SELECT newuuid(1) FROM 't' | newuuid",
+ "SELECT isNull() AS n FROM 't' | isNull",
+ "SELECT isNull(a, b) AS n FROM 't' | isNull",
+ "SELECT isUndefined() AS u FROM 't' | isUndefined",
+ "SELECT timestamp FROM 't' WHERE timestamp( | ''"
+ })
+ void rejectsMalformedUsesNamingTheToken(String sql, String token) {
+ RuleSqlParseException failure = assertThrows(RuleSqlParseException.class, () -> RuleSqlParser.parse(sql));
+
+ assertEquals(token, failure.token(), failure.getMessage());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParserTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParserTest.java
index 027f69ee0a..3967567bca 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParserTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/rules/RuleSqlParserTest.java
@@ -143,16 +143,16 @@ class RuleSqlParserTest {
@Test
void reportsTheOffendingTokenAndPositionOfAnUnsupportedFunction() {
RuleSqlParseException failure = assertThrows(RuleSqlParseException.class,
- () -> RuleSqlParser.parse("SELECT clientid() FROM 'a/b'"));
+ () -> RuleSqlParser.parse("SELECT principal() FROM 'a/b'"));
- assertEquals("clientid", failure.token());
+ assertEquals("principal", failure.token());
assertEquals(7, failure.position());
- assertTrue(failure.getMessage().contains("clientid"), failure.getMessage());
+ assertTrue(failure.getMessage().contains("principal"), failure.getMessage());
}
@ParameterizedTest
@CsvSource(delimiter = '|', value = {
- "SELECT * FROM 'a/b' WHERE state IN ('on') | IN",
+ "SELECT * FROM 'a/b' WHERE state IN ('on') | (",
"SELECT * FROM 'a/b' WHERE state IS NULL | IS",
"SELECT CASE WHEN a THEN 1 END FROM 'a/b' | WHEN",
"SELECT * FROM 'a/b' WHERE encode(a, 'base64') = 'x'| encode",