Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(iot): search the supported fleet index subset (#4913)

Mario Toffia1082941bc20d
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/IotTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/IotTest.java
index 2ede66e98f..bdc0d4a57e 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/IotTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/IotTest.java
@@ -63,6 +63,7 @@ import software.amazon.awssdk.services.iot.model.GetPolicyRequest;
import software.amazon.awssdk.services.iot.model.GetPolicyResponse;
import software.amazon.awssdk.services.iot.model.GetTopicRuleRequest;
import software.amazon.awssdk.services.iot.model.GetTopicRuleResponse;
+import software.amazon.awssdk.services.iot.model.IndexNotReadyException;
import software.amazon.awssdk.services.iot.model.IndexStatus;
import software.amazon.awssdk.services.iot.model.InvalidRequestException;
import software.amazon.awssdk.services.iot.model.ListCertificatesRequest;
@@ -88,17 +89,21 @@ import software.amazon.awssdk.services.iot.model.NamedShadowIndexingMode;
import software.amazon.awssdk.services.iot.model.RemoveThingFromThingGroupRequest;
import software.amazon.awssdk.services.iot.model.ResourceAlreadyExistsException;
import software.amazon.awssdk.services.iot.model.ResourceNotFoundException;
+import software.amazon.awssdk.services.iot.model.SearchIndexRequest;
+import software.amazon.awssdk.services.iot.model.SearchIndexResponse;
import software.amazon.awssdk.services.iot.model.ServerCertificateStatus;
import software.amazon.awssdk.services.iot.model.ServiceType;
import software.amazon.awssdk.services.iot.model.SqsAction;
import software.amazon.awssdk.services.iot.model.Tag;
import software.amazon.awssdk.services.iot.model.TagResourceRequest;
import software.amazon.awssdk.services.iot.model.ThingConnectivityIndexingMode;
+import software.amazon.awssdk.services.iot.model.ThingDocument;
import software.amazon.awssdk.services.iot.model.ThingGroupIndexingConfiguration;
import software.amazon.awssdk.services.iot.model.ThingGroupIndexingMode;
import software.amazon.awssdk.services.iot.model.ThingGroupProperties;
import software.amazon.awssdk.services.iot.model.ThingIndexingConfiguration;
import software.amazon.awssdk.services.iot.model.ThingIndexingMode;
+import software.amazon.awssdk.services.iot.model.ThingTypeDefinition;
import software.amazon.awssdk.services.iot.model.ThingTypeProperties;
import software.amazon.awssdk.services.iot.model.TopicRulePayload;
import software.amazon.awssdk.services.iot.model.UntagResourceRequest;
@@ -140,8 +145,14 @@ import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -359,6 +370,173 @@ class IotTest {
}
}
+ @Test
+ void fleetIndexSearch() throws InterruptedException {
+ String prefix = "fleetsearch" + Long.toString(System.currentTimeMillis(), 36);
+ String group = prefix + "group";
+ String provider = prefix + "p";
+ // AWS keeps a deprecated thing type for five minutes before it can be deleted, so on AWS the
+ // test borrows a test thing type the account already has rather than creating one.
+ String thingType = TestFixtures.isRealAws() ? existingTestThingType() : prefix + "type";
+ GetIndexingConfigurationResponse prior =
+ iot.getIndexingConfiguration(GetIndexingConfigurationRequest.builder().build());
+ try {
+ iot.updateIndexingConfiguration(UpdateIndexingConfigurationRequest.builder()
+ .thingIndexingConfiguration(ThingIndexingConfiguration.builder()
+ .thingIndexingMode(ThingIndexingMode.REGISTRY)
+ .thingConnectivityIndexingMode(ThingConnectivityIndexingMode.STATUS)
+ .build())
+ .build());
+ if (!TestFixtures.isRealAws()) {
+ iot.createThingType(CreateThingTypeRequest.builder().thingTypeName(thingType).build());
+ }
+ iot.createThingGroup(CreateThingGroupRequest.builder().thingGroupName(group).build());
+ iot.createThing(CreateThingRequest.builder()
+ .thingName(prefix + "a")
+ .thingTypeName(thingType)
+ .attributePayload(AttributePayload.builder()
+ .attributes(Map.of("provider", provider + "1", "site", "north"))
+ .build())
+ .build());
+ iot.createThing(CreateThingRequest.builder()
+ .thingName(prefix + "b")
+ .attributePayload(AttributePayload.builder().attributes(Map.of("provider", provider + "1")).build())
+ .build());
+ iot.createThing(CreateThingRequest.builder()
+ .thingName(prefix + "c")
+ .attributePayload(AttributePayload.builder().attributes(Map.of("provider", provider + "2")).build())
+ .build());
+ iot.addThingToThingGroup(AddThingToThingGroupRequest.builder()
+ .thingGroupName(group)
+ .thingName(prefix + "a")
+ .build());
+
+ List<ThingDocument> all = awaitSearch("thingName:" + prefix + "*", prefix, "a", "b", "c");
+ ThingDocument typed = all.get(0);
+ assertThat(typed.thingTypeName()).isEqualTo(thingType);
+ assertThat(typed.thingGroupNames()).containsExactly(group);
+ assertThat(typed.attributes()).containsOnly(Map.entry("provider", provider + "1"), Map.entry("site", "north"));
+ assertThat(typed.shadow()).isNull();
+ ThingDocument plain = all.get(1);
+ assertThat(plain.thingTypeName()).isNull();
+ assertThat(plain.hasThingGroupNames()).isFalse();
+ assertThat(plain.attributes()).containsOnly(Map.entry("provider", provider + "1"));
+ for (ThingDocument thing : all) {
+ assertThat(thing.thingId()).isNotBlank();
+ }
+
+ awaitSearch("attributes.provider:" + provider + "1", prefix, "a", "b");
+ awaitSearch("thingName:" + prefix + "c", prefix, "c");
+ if (thingType != null) {
+ awaitSearch("thingTypeName:" + thingType, prefix, "a");
+ awaitSearch("thingTypeName:" + thingType + " AND attributes.provider:" + provider + "1", prefix, "a");
+ }
+ awaitSearch("attributes.provider:" + provider + "2 OR thingName:" + prefix + "a", prefix, "a", "c");
+ awaitSearch("thingName:" + prefix + "* AND NOT attributes.provider:" + provider + "1", prefix, "c");
+ awaitSearch("attributes.provider:" + provider + "?", prefix, "a", "b", "c");
+ awaitSearch("thingGroupNames:" + group, prefix, "a");
+
+ Set<String> paged = new TreeSet<>();
+ String nextToken = null;
+ int pages = 0;
+ do {
+ SearchIndexResponse page = iot.searchIndex(SearchIndexRequest.builder()
+ .queryString("thingName:" + prefix + "*")
+ .maxResults(1)
+ .nextToken(nextToken)
+ .build());
+ assertThat(page.things()).hasSizeLessThanOrEqualTo(1);
+ page.things().forEach(thing -> paged.add(thing.thingName()));
+ nextToken = page.nextToken();
+ pages++;
+ } while (nextToken != null && pages < 10);
+ assertThat(paged).containsExactly(prefix + "a", prefix + "b", prefix + "c");
+ assertThat(pages).isBetween(3, 4);
+
+ assertThatThrownBy(() -> iot.searchIndex(SearchIndexRequest.builder()
+ .queryString("thingName:" + prefix + "*")
+ .nextToken("bogus")
+ .build()))
+ .isInstanceOf(InvalidRequestException.class)
+ .hasMessageContaining("Invalid nextToken");
+ assertThatThrownBy(() -> iot.searchIndex(SearchIndexRequest.builder()
+ .queryString("thingName:" + prefix + "*")
+ .queryVersion("2017-09-31")
+ .build()))
+ .isInstanceOf(InvalidRequestException.class)
+ .hasMessageContaining("Invalid queryVersion. Expected one of: [2017-09-30]");
+ } finally {
+ for (String suffix : List.of("a", "b", "c")) {
+ deleteIfPresent(() -> iot.deleteThing(DeleteThingRequest.builder().thingName(prefix + suffix).build()));
+ }
+ deleteIfPresent(() -> iot.deleteThingGroup(DeleteThingGroupRequest.builder().thingGroupName(group).build()));
+ if (!TestFixtures.isRealAws()) {
+ deleteIfPresent(() -> iot.deleteThingType(DeleteThingTypeRequest.builder().thingTypeName(thingType).build()));
+ }
+ // Indexing is account wide on AWS: put back whatever the account had before this test.
+ iot.updateIndexingConfiguration(UpdateIndexingConfigurationRequest.builder()
+ .thingIndexingConfiguration(prior.thingIndexingConfiguration())
+ .thingGroupIndexingConfiguration(prior.thingGroupIndexingConfiguration())
+ .build());
+ }
+ }
+
+ /** A thing type of the account meant for tests and not deprecated, or null when it has none. */
+ private String existingTestThingType() {
+ return iot.listThingTypesPaginator(ListThingTypesRequest.builder().build()).thingTypes().stream()
+ .filter(type -> !Boolean.TRUE.equals(type.thingTypeMetadata().deprecated()))
+ .map(ThingTypeDefinition::thingTypeName)
+ .filter(name -> name.contains("test"))
+ .sorted()
+ .findFirst()
+ .orElse(null);
+ }
+
+ /**
+ * This test's things a query finds, in name order. On AWS the search is polled until they are the
+ * expected ones: AWS indexes a change a few seconds after it and refuses searches while a new
+ * index builds. Floci answers at once, so against Floci the search runs once.
+ */
+ private List<ThingDocument> awaitSearch(String query, String prefix, String... expectedSuffixes)
+ throws InterruptedException {
+ List<String> expected = Arrays.stream(expectedSuffixes).map(suffix -> prefix + suffix).toList();
+ Instant deadline = Instant.now().plusSeconds(120);
+ while (true) {
+ List<ThingDocument> found = new ArrayList<>();
+ String notReady = null;
+ try {
+ String nextToken = null;
+ do {
+ SearchIndexResponse page = iot.searchIndex(SearchIndexRequest.builder()
+ .queryString(query)
+ .nextToken(nextToken)
+ .build());
+ page.things().stream().filter(thing -> thing.thingName().startsWith(prefix)).forEach(found::add);
+ nextToken = page.nextToken();
+ } while (nextToken != null);
+ } catch (IndexNotReadyException e) {
+ notReady = e.getMessage();
+ }
+ found.sort(Comparator.comparing(ThingDocument::thingName));
+ List<String> names = found.stream().map(ThingDocument::thingName).toList();
+ if (!TestFixtures.isRealAws() || notReady == null && names.equals(expected)
+ || Instant.now().isAfter(deadline)) {
+ assertThat(notReady).as("index not ready").isNull();
+ assertThat(names).as(query).isEqualTo(expected);
+ return found;
+ }
+ Thread.sleep(2_000);
+ }
+ }
+
+ private static void deleteIfPresent(Runnable delete) {
+ try {
+ delete.run();
+ } catch (ResourceNotFoundException ignored) {
+ // The test failed before creating it.
+ }
+ }
+
private static void assertIndexStatus(IndexStatus status) {
if (TestFixtures.isRealAws()) {
// AWS builds the index in the background after a mode change; Floci has it ready at once.
diff --git a/docs/services/index.md b/docs/services/index.md
index 750246c56d..be3277207b 100644
--- a/docs/services/index.md
+++ b/docs/services/index.md
@@ -142,7 +142,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac
| [BCM Data Exports](bcm-data-exports.md) | `POST /` + `X-Amz-Target: AWSBillingAndCostManagementDataExports.*` | JSON 1.1 | 7 |
| [Transfer Family](transfer.md) | `POST /` + `X-Amz-Target: TransferService.*` | JSON 1.1 | 17 |
| [DataSync](datasync.md) | `POST /` + `X-Amz-Target: FmrsService.*` | JSON 1.1 | 48 |
-| [IoT Core](iot.md) | `/things/...`, `/endpoint`, rules/policies REST paths | REST JSON | 74 |
+| [IoT Core](iot.md) | `/things/...`, `/endpoint`, rules/policies REST paths | REST JSON | 75 |
| [IoT Data](iot.md) | `/things/{thingName}/shadow`, MQTT topics | REST JSON | 11 |
**Lambda, ElastiCache, RDS, MSK, MWAA, ECS, EKS, and OpenSearch** spin up real Docker containers and support IAM authentication and SigV4 request signing, the same auth flow as production AWS. **RDS Data API** executes SQL against the local RDS containers through AWS-compatible REST JSON routes.
diff --git a/docs/services/iot.md b/docs/services/iot.md
index a156363fee..4c2dd781e7 100644
--- a/docs/services/iot.md
+++ b/docs/services/iot.md
@@ -45,7 +45,7 @@ Current MVP 2 limitations:
- `SendDirectMessage` publishes to the requested MQTT topic through the embedded broker. Unlike AWS IoT Core, it does not yet bypass subscription matching to deliver to a client that is not subscribed to that topic.
- `GetConnection` and `ListSubscriptions` report live in-memory broker state only; offline persistent session subscription reporting is not modeled yet.
- Jobs reserved MQTT topics remain follow-up scope; Jobs Data HTTP APIs are implemented first.
-- Dynamic thing groups, job rollouts, cancellations, documents from S3, and advanced job scheduling are not yet modeled. Fleet indexing has its configuration only, see [Fleet Indexing](#fleet-indexing).
+- Dynamic thing groups, job rollouts, cancellations, documents from S3, and advanced job scheduling are not yet modeled. Fleet indexing has its configuration and a bounded `SearchIndex` subset, see [Fleet Indexing](#fleet-indexing).
## Domain Configurations
@@ -104,7 +104,7 @@ Current limitations:
## Fleet Indexing
-Status: configuration only.
+Status: configuration and a bounded SearchIndex subset.
`UpdateIndexingConfiguration` and `GetIndexingConfiguration` (`/indexing/config`) and `DescribeIndex` (`/indices/{indexName}`) are served on the REST-JSON paths the AWS SDKs use, with the AWS shapes, error codes and validation messages:
@@ -115,9 +115,50 @@ Status: configuration only.
- A member of the wrong JSON type, or content after the JSON body, fails with `SerializationException`.
- `DescribeIndex` reports `AWS_Things` and `AWS_ThingGroups` with the schema the modes select. The index is `ACTIVE` as soon as it is enabled: Floci has no `BUILDING` or `REBUILDING` window. A disabled index is `ResourceNotFoundException`, any other index name `InvalidRequestException`.
+### SearchIndex
+
+`SearchIndex` (`POST /indices/search`) searches the things of the caller's account and region in the `AWS_Things` index, the default index name. Floci evaluates this part of the query language:
+
+- `field:value`. Values match case-insensitively, field names exactly: `attributes.Site:north` matches nothing when the attribute is named `site`.
+- `*` (any run of characters) and `?` (one character) inside a value, and `\` to escape one character. A quoted value such as `thingName:"x"` matches literally.
+- `field:*` matches the things that have the field. A bare `*` matches every thing.
+- `AND`, `OR` and `NOT` in upper case, their forms `&&`, `||` and `!`, a leading `-` for negation, and parentheses.
+- Precedence as measured on AWS: `NOT` binds tightest, then `AND`, then `OR`. Whitespace is an `AND` that binds looser than `OR`, so `a OR b c` means `(a OR b) AND c`.
+
+The fields are `thingName`, `thingId`, `thingTypeName`, `thingGroupNames` (direct memberships) and `attributes.<name>`.
+
+Errors follow AWS, with AWS's messages. The body and member types are checked first, then the member constraints, then the index, then the query, then `nextToken`:
+
+- `InvalidRequestException` for a missing or empty `queryString`, a `queryString` longer than 1000 characters, a `maxResults` below 1 (AWS truncates it toward zero and caps it at the 32-bit integer range), an empty `indexName`, a `queryVersion` other than `2017-09-30`, an unknown index name, a `nextToken` that is empty, was not issued by `SearchIndex`, or was edited or truncated (`Invalid nextToken`), and a `nextToken` issued for a different `queryString`, `indexName` or `queryVersion` (`nextToken was from a request with different parameters`).
+- `ResourceNotFoundException` for an index that is not enabled.
+- `InvalidQueryException` for invalid syntax, an invalid field name (including a `shadow.name.<shadow>.<field>` whose shadow is not in the named shadow filter), more than 12 terms, more than two `*` wildcards in a value, a value starting with `?`, the `+` operator, and fuzzy, regular expression and boost queries.
+- `InvalidRequestException` for a `connectivity.*`, `shadow.*` or `deviceDefender.*` field while that indexing is off. A classic `shadow.*` field needs `REGISTRY_AND_SHADOW`; a `shadow.name.*` field is also served by named shadow indexing.
+- `SerializationException` for a body that is not a JSON object, malformed JSON or content after it, a `queryString`, `indexName`, `queryVersion` or `nextToken` that is not a string, and a `maxResults` that is not a number.
+
+Within the query, invalid syntax is reported before more than 12 terms, and more than 12 terms before any other query error, including a `connectivity.*`, `shadow.*` or `deviceDefender.*` field while that indexing is off and the constructs Floci refuses below. Each value of a field group counts as a term, the field before it does not. The field of a group is checked like any other field, before the group is refused. A range is one term: two bounds, separated by spaces and optionally by `TO`, in `[` or `{` and `]` or `}`. An unclosed or otherwise malformed range, and an unescaped `[`, `]`, `{` or `}` outside a range, is invalid syntax, as on AWS.
+
+Syntax AWS accepts but Floci does not evaluate is refused with `InvalidQueryException` and the message `Floci does not support <construct> in fleet index queries, query string: <query>`. It never answers with an empty result. This covers:
+
+- range queries (`field:[a TO b]`, `field:{a TO b}`)
+- comparisons (`>`, `<`, `>=`, `<=`)
+- free text terms without a field, including lower case `and`, `or` and `not`
+- field grouping (`field:(a OR b)`)
+- every `connectivity.*`, `shadow.*` and `deviceDefender.*` field once its indexing is on. While that indexing is off the answer is AWS's `InvalidRequestException` above.
+
+Searching `AWS_ThingGroups` is refused with `InvalidRequestException` and `Floci does not support searching AWS_ThingGroups` while that index is enabled. While it is disabled the answer is AWS's `ResourceNotFoundException`.
+
+Results:
+
+- A thing has `thingName` and `thingId`, and `thingTypeName`, `thingGroupNames` and `attributes` only when it has them. `shadow` and `deviceDefender` are never returned: Floci keeps no indexed shadow document.
+- `connectivity` is never returned either. AWS adds it while `thingConnectivityIndexingMode` is `STATUS`, where a thing that never connected shows `connected` `false` and `timestamp` `0`.
+- Things come in thing name order. AWS does not specify an order.
+- `maxResults` and `nextToken` page the results. Without `maxResults` every match comes in one page. The token marks the last thing returned, so the next page continues after it even when things before it were deleted, as on AWS. A token continues only the request that issued it: the same `queryString`, and `indexName` and `queryVersion` sent the same way, since an omitted member differs from its explicit default. `maxResults` may change between pages.
+- An index answers at once: a thing is searchable as soon as it is created, where AWS takes a few seconds.
+
Current limitations:
-- `SearchIndex`, `ListIndices` and the statistics and aggregation APIs (`GetStatistics`, `GetCardinality`, `GetPercentiles`, `GetBucketsAggregation`) are not modeled yet.
+- `ListIndices` and the statistics and aggregation APIs (`GetStatistics`, `GetCardinality`, `GetPercentiles`, `GetBucketsAggregation`) are not modeled yet.
+- `customFields` types are not applied to search: attribute values are matched as strings.
- `managedFields` and `customFields` sent in the thing group configuration are checked (the custom fields for their types only) but not stored: `GetIndexingConfiguration` reports the managed fields the mode selects and no custom fields.
## MQTT Broker
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexQuery.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexQuery.java
new file mode 100644
index 0000000000..608100d8b8
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexQuery.java
@@ -0,0 +1,381 @@
+package io.github.hectorvent.floci.services.iot;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.iot.model.IotIndexingConfiguration.ThingIndexing;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * The part of the AWS IoT fleet indexing query language Floci evaluates, parsed into a predicate
+ * over a thing's index document. Precedence is the one measured on AWS: NOT binds tightest, then
+ * AND, then OR, and whitespace is an AND looser than OR, so {@code a OR b c} is
+ * {@code (a OR b) AND c}. Values match case-insensitively, field names exactly. Floci evaluates the
+ * registry fields and attributes; connectivity, shadow and Device Defender fields are refused, with
+ * AWS's error while their indexing is off. Syntax AWS accepts but Floci does not evaluate is refused
+ * rather than matching nothing. A query is checked in the order measured on AWS: syntax, then the
+ * 12-term limit, then fields and values, the first such error in the query winning.
+ */
+final class IotFleetIndexQuery {
+
+ private static final Set<String> FIELDS = Set.of("thingName", "thingId", "thingTypeName", "thingGroupNames");
+ private static final String ATTRIBUTES = "attributes.";
+ private static final Set<String> OPERATORS = Set.of("AND", "OR", "NOT", "-", ")");
+ private static final Pattern COMPARISON = Pattern.compile("[<>]");
+ private static final String RANGE_BOUND = "(\"(?:\\\\.|[^\"\\\\])*\"|[^ \\]}]+)";
+ private static final Pattern RANGE = Pattern.compile("[\\[{] *" + RANGE_BOUND + " +(?:TO +)?" + RANGE_BOUND
+ + " *[\\]}]");
+ private static final int MAX_TERMS = 12;
+ private static final int MAX_WILDCARDS = 2;
+
+ private final String queryString;
+ private final ThingIndexing indexing;
+ private final List<String> tokens;
+ private int position;
+ private int terms;
+ private AwsException deferred;
+
+ private IotFleetIndexQuery(String queryString, ThingIndexing indexing) {
+ this.queryString = queryString;
+ this.indexing = indexing;
+ this.tokens = tokenize();
+ }
+
+ static Predicate<JsonNode> parse(String queryString, ThingIndexing indexing) {
+ IotFleetIndexQuery parser = new IotFleetIndexQuery(queryString, indexing);
+ Predicate<JsonNode> query = parser.clauses();
+ if (parser.position < parser.tokens.size()) {
+ throw parser.syntaxError();
+ }
+ if (parser.terms > MAX_TERMS) {
+ throw parser.parseError("number of terms cannot exceed " + MAX_TERMS + ", terms found: " + parser.terms);
+ }
+ if (parser.deferred != null) {
+ throw parser.deferred;
+ }
+ return query;
+ }
+
+ /**
+ * Words, quoted runs kept inside their word, parentheses, and a leading - or ! as negation. A
+ * range opened right after a field's colon, or at the start of a word, runs to its closing
+ * bracket as one word.
+ */
+ private List<String> tokenize() {
+ List<String> found = new ArrayList<>();
+ int i = 0;
+ while (i < queryString.length()) {
+ char c = queryString.charAt(i);
+ if (Character.isWhitespace(c)) {
+ i++;
+ } else if (c == '(' || c == ')') {
+ found.add(String.valueOf(c));
+ i++;
+ } else if (c == '-' || c == '!') {
+ found.add("-");
+ i++;
+ } else {
+ int start = i;
+ boolean quoted = false;
+ boolean range = false;
+ while (i < queryString.length()) {
+ char next = queryString.charAt(i);
+ if (next == '\\') {
+ i++;
+ } else if (next == '"') {
+ quoted = !quoted;
+ } else if (!quoted && range && (next == ']' || next == '}')) {
+ i++;
+ break;
+ } else if (!quoted && !range && (next == '[' || next == '{')
+ && (i == start || queryString.charAt(i - 1) == ':')) {
+ range = true;
+ } else if (!quoted && !range && (Character.isWhitespace(next) || next == '(' || next == ')')) {
+ break;
+ }
+ i++;
+ }
+ if (quoted) {
+ throw syntaxError();
+ }
+ String word = queryString.substring(start, Math.min(i, queryString.length()));
+ found.add(switch (word) {
+ case "&&" -> "AND";
+ case "||" -> "OR";
+ default -> word;
+ });
+ }
+ }
+ return found;
+ }
+
+ /** Clauses joined by whitespace, which AWS treats as an AND binding looser than OR. */
+ private Predicate<JsonNode> clauses() {
+ Predicate<JsonNode> query = or();
+ while (position < tokens.size() && !")".equals(tokens.get(position))) {
+ query = query.and(or());
+ }
+ return query;
+ }
+
+ private Predicate<JsonNode> or() {
+ Predicate<JsonNode> query = and();
+ while (accept("OR")) {
+ query = query.or(and());
+ }
+ return query;
+ }
+
+ private Predicate<JsonNode> and() {
+ Predicate<JsonNode> query = unary();
+ while (accept("AND")) {
+ query = query.and(unary());
+ }
+ return query;
+ }
+
+ private Predicate<JsonNode> unary() {
+ if (accept("NOT") || accept("-")) {
+ return primary().negate();
+ }
+ return primary();
+ }
+
+ private Predicate<JsonNode> primary() {
+ if (position >= tokens.size()) {
+ throw syntaxError();
+ }
+ String token = tokens.get(position++);
+ if ("(".equals(token)) {
+ Predicate<JsonNode> group = clauses();
+ if (!accept(")")) {
+ throw syntaxError();
+ }
+ return group;
+ }
+ if (OPERATORS.contains(token)) {
+ throw syntaxError();
+ }
+ return term(token);
+ }
+
+ private Predicate<JsonNode> term(String token) {
+ int colon = token.indexOf(':');
+ String value = colon < 0 ? "" : token.substring(colon + 1);
+ if (colon >= 0 && value.isEmpty()) {
+ if (hasBracket(token) || !accept("(")) {
+ throw syntaxError();
+ }
+ // AWS counts each value of a field group as a term, but not the field, and checks the field first.
+ Predicate<JsonNode> refused;
+ try {
+ checkField(token.substring(0, colon));
+ refused = defer(unsupported("field grouping"));
+ } catch (AwsException e) {
+ refused = defer(e);
+ }
+ clauses();
+ if (!accept(")")) {
+ throw syntaxError();
+ }
+ return refused;
+ }
+ terms++;
+ boolean quoted = value.startsWith("\"");
+ String rangeText = colon < 0 ? token : value;
+ boolean range = rangeText.startsWith("[") || rangeText.startsWith("{");
+ if ((quoted && (value.length() < 2 || !value.endsWith("\"")))
+ || (!quoted && (value.startsWith("<") || value.startsWith(">")))
+ || hasBracket(range ? token.substring(0, token.length() - rangeText.length()) : token)
+ || (range && !isRange(rangeText))) {
+ throw syntaxError();
+ }
+ try {
+ return match(token, colon, value);
+ } catch (AwsException e) {
+ return defer(e);
+ }
+ }
+
+ private Predicate<JsonNode> match(String token, int colon, String value) {
+ if ("*".equals(token)) {
+ return document -> true;
+ }
+ if (token.startsWith("+")) {
+ throw parseError("unsupported operator - \"+\"");
+ }
+ Matcher comparison = COMPARISON.matcher(token);
+ if (comparison.find() && (colon < 0 || comparison.start() < colon)) {
+ checkField(token.substring(0, comparison.start()));
+ throw unsupported("comparisons");
+ }
+ if (colon < 0) {
+ throw unsupported("free text terms");
+ }
+ String field = token.substring(0, colon);
+ checkField(field);
+ Pattern pattern = valuePattern(field, value);
+ return document -> {
+ int dot = field.indexOf('.');
+ JsonNode node = dot < 0 ? document.path(field)
+ : document.path(field.substring(0, dot)).path(field.substring(dot + 1));
+ Iterable<JsonNode> values = node.isArray() ? node : node.isMissingNode() ? List.of() : List.of(node);
+ for (JsonNode candidate : values) {
+ if (pattern.matcher(candidate.asText()).matches()) {
+ return true;
+ }
+ }
+ return false;
+ };
+ }
+
+ /**
+ * A registry field or attribute passes. One under connectivity, shadow or Device Defender first
+ * needs that indexing on, with AWS's error: a classic shadow field needs REGISTRY_AND_SHADOW, a
+ * named shadow field needs named shadow indexing and a shadow in its filter (otherwise an invalid
+ * field name, as on AWS). Floci then refuses it, since it indexes none of them. Anything else is
+ * the invalid field name AWS reports.
+ */
+ private void checkField(String field) {
+ if (field.startsWith(ATTRIBUTES) && field.length() > ATTRIBUTES.length()) {
+ return;
+ }
+ String attribute;
+ boolean enabled;
+ if (field.startsWith("connectivity.")) {
+ attribute = "Connectivity";
+ enabled = "STATUS".equals(indexing.thingConnectivityIndexingMode());
+ } else if (field.startsWith("shadow.")) {
+ attribute = "Shadow";
+ boolean named = "shadow.name".equals(field) || field.startsWith("shadow.name.");
+ enabled = "REGISTRY_AND_SHADOW".equals(indexing.thingIndexingMode())
+ || named && "ON".equals(indexing.namedShadowIndexingMode());
+ if (enabled && named) {
+ // shadow.name.<shadow>.<field> exists only for a shadow in the named shadow filter.
+ String[] parts = field.split("\\.", 4);
+ if (parts.length < 4 || !"ON".equals(indexing.namedShadowIndexingMode())
+ || !indexing.namedShadowNames().contains(parts[2])) {
+ throw parseError("invalid field name, field name: " + field);
+ }
+ }
+ } else if (field.startsWith("deviceDefender.")) {
+ attribute = "Devicedefender";
+ enabled = "VIOLATIONS".equals(indexing.deviceDefenderIndexingMode());
+ } else if (FIELDS.contains(field)) {
+ return;
+ } else {
+ throw parseError("invalid field name, field name: " + field);
+ }
+ if (!enabled) {
+ throw new AwsException("InvalidRequestException", "Query includes one or more constraints for "
+ + attribute + " attribute, but " + attribute + " indexing is not enabled for AWS_Things index", 400);
+ }
+ if (!FIELDS.contains(field)) {
+ throw unsupported("the field " + field);
+ }
+ }
+
+ /**
+ * A quoted value matches literally; otherwise * and ? are wildcards and a backslash escapes.
+ * As on AWS an unquoted value takes at most two unescaped * and cannot start with an unescaped ?.
+ */
+ private Pattern valuePattern(String field, String value) {
+ boolean quoted = value.startsWith("\"");
+ if (!quoted) {
+ switch (value.charAt(0)) {
+ case '[', '{' -> throw unsupported("range queries");
+ case '/' -> throw parseError("unsupported query type - regular expression");
+ case '?' -> throw invalidValue("wildcard character (?) is not allowed at the start of the value",
+ field, value);
+ default -> {
+ }
+ }
+ }
+ String text = quoted ? value.substring(1, value.length() - 1) : value;
+ StringBuilder regex = new StringBuilder();
+ int wildcards = 0;
+ for (int i = 0; i < text.length(); i++) {
+ char c = text.charAt(i);
+ if (c == '\\' && i + 1 < text.length()) {
+ regex.append(Pattern.quote(String.valueOf(text.charAt(++i))));
+ } else if (!quoted && c == '*') {
+ wildcards++;
+ regex.append(".*");
+ } else if (!quoted && c == '?') {
+ regex.append('.');
+ } else if (!quoted && c == '~') {
+ throw parseError("unsupported query type - fuzzy");
+ } else if (!quoted && c == '^') {
+ throw parseError("unsupported query type - boost");
+ } else {
+ regex.append(Pattern.quote(String.valueOf(c)));
+ }
+ }
+ if (wildcards > MAX_WILDCARDS) {
+ throw invalidValue("too many wildcards (*). The maximum allowed per term is " + MAX_WILDCARDS,
+ field, value);
+ }
+ return Pattern.compile(regex.toString(), Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.DOTALL);
+ }
+
+ /** A [, ], { or } that is neither escaped nor quoted, which AWS's parser rejects outside a range. */
+ private static boolean hasBracket(String text) {
+ boolean quoted = false;
+ for (int i = 0; i < text.length(); i++) {
+ char c = text.charAt(i);
+ if (c == '\\') {
+ i++;
+ } else if (c == '"') {
+ quoted = !quoted;
+ } else if (!quoted && "[]{}".indexOf(c) >= 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** Two bounds, optionally joined by TO, in [ or { and ] or }, where TO itself is no bound. */
+ private static boolean isRange(String text) {
+ Matcher range = RANGE.matcher(text);
+ return range.matches() && !"TO".equals(range.group(1)) && !"TO".equals(range.group(2));
+ }
+
+ /** Records the first error AWS reports only once the query parses and its terms are counted. */
+ private Predicate<JsonNode> defer(AwsException error) {
+ if (deferred == null) {
+ deferred = error;
+ }
+ return document -> false;
+ }
+
+ private boolean accept(String token) {
+ if (position < tokens.size() && token.equals(tokens.get(position))) {
+ position++;
+ return true;
+ }
+ return false;
+ }
+
+ private AwsException syntaxError() {
+ return parseError("invalid syntax");
+ }
+
+ private AwsException parseError(String reason) {
+ return new AwsException("InvalidQueryException",
+ "Unable to parse query, " + reason + ", query string: " + queryString, 400);
+ }
+
+ private AwsException invalidValue(String reason, String field, String value) {
+ return parseError("invalid value - " + reason + ", field name: " + field + ", value: " + value);
+ }
+
+ private AwsException unsupported(String construct) {
+ return new AwsException("InvalidQueryException",
+ "Floci does not support " + construct + " in fleet index queries, query string: " + queryString, 400);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingController.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingController.java
index 6a1aedf14e..8b84b49a16 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingController.java
@@ -22,7 +22,7 @@ import java.util.List;
/**
* REST-JSON routes for AWS IoT fleet indexing: UpdateIndexingConfiguration,
- * GetIndexingConfiguration and DescribeIndex, on the paths and shapes the AWS SDKs use.
+ * GetIndexingConfiguration, DescribeIndex and SearchIndex, on the paths and shapes the AWS SDKs use.
*/
@Path("/")
@Produces(MediaType.APPLICATION_JSON)
@@ -93,4 +93,17 @@ public class IotFleetIndexingController {
response.put("schema", schema);
return Response.ok(response).build();
}
+
+ @POST
+ @Path("/indices/search")
+ public Response searchIndex(@Context HttpHeaders headers, String body) {
+ IotService.Page<ObjectNode> page = fleetIndexingService.searchIndex(IotRequestBody.read(objectMapper, body),
+ regionResolver.resolveRegion(headers));
+ ObjectNode response = objectMapper.createObjectNode();
+ response.putArray("things").addAll(page.items());
+ if (page.nextToken() != null) {
+ response.put("nextToken", page.nextToken());
+ }
+ return Response.ok(response).build();
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingService.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingService.java
index ff4850ab67..659b1e6284 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingService.java
@@ -2,8 +2,10 @@ package io.github.hectorvent.floci.services.iot;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.JsonNodeType;
import com.fasterxml.jackson.databind.node.MissingNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
@@ -11,21 +13,30 @@ import io.github.hectorvent.floci.services.iot.model.IotIndexingConfiguration;
import io.github.hectorvent.floci.services.iot.model.IotIndexingConfiguration.Field;
import io.github.hectorvent.floci.services.iot.model.IotIndexingConfiguration.GeoLocation;
import io.github.hectorvent.floci.services.iot.model.IotIndexingConfiguration.ThingIndexing;
+import io.github.hectorvent.floci.services.iot.model.IotThingGroup;
+import io.github.hectorvent.floci.services.iot.model.Thing;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Base64;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
+import java.util.function.Predicate;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
- * AWS IoT fleet indexing configuration: UpdateIndexingConfiguration, GetIndexingConfiguration and
- * DescribeIndex. One configuration per account and region; a region never updated is OFF.
+ * AWS IoT fleet indexing: UpdateIndexingConfiguration, GetIndexingConfiguration, DescribeIndex and
+ * a bounded SearchIndex. One configuration per account and region; a region never updated is OFF.
*/
@ApplicationScoped
public class IotFleetIndexingService {
@@ -33,6 +44,9 @@ public class IotFleetIndexingService {
static final String THINGS_INDEX = "AWS_Things";
static final String THING_GROUPS_INDEX = "AWS_ThingGroups";
+ private static final int MAX_QUERY_STRING_LENGTH = 1000;
+ private static final Pattern PAGE_TOKEN = Pattern.compile("([0-9a-f-]{36})([0-9a-f-]{36})(.+)", Pattern.DOTALL);
+
// Enum value sets in the order AWS prints them in its validation messages.
private static final List<String> THING_INDEXING_MODES = List.of("REGISTRY_AND_SHADOW", "OFF", "REGISTRY");
private static final List<String> CONNECTIVITY_MODES = List.of("OFF", "STATUS");
@@ -81,17 +95,19 @@ public class IotFleetIndexingService {
new Field("thingGroupId", "String"));
private final StorageBackend<String, IotIndexingConfiguration> store;
+ private final IotService iotService;
/** Guards the read-modify-write of an update that changes only one of the two configurations. */
private final Object lock = new Object();
@Inject
- public IotFleetIndexingService(StorageFactory storageFactory) {
+ public IotFleetIndexingService(StorageFactory storageFactory, IotService iotService) {
this(storageFactory.create("iot", "iot-indexing-configuration.json",
- new TypeReference<Map<String, IotIndexingConfiguration>>() {}));
+ new TypeReference<Map<String, IotIndexingConfiguration>>() {}), iotService);
}
- IotFleetIndexingService(StorageBackend<String, IotIndexingConfiguration> store) {
+ IotFleetIndexingService(StorageBackend<String, IotIndexingConfiguration> store, IotService iotService) {
this.store = store;
+ this.iotService = iotService;
}
public void updateIndexingConfiguration(JsonNode request, String region) {
@@ -114,10 +130,7 @@ public class IotFleetIndexingService {
List<Field> groupManagedFields = fields(groupNode.path("managedFields"), groupPrefix + ".managedFields",
errors);
fields(groupNode.path("customFields"), groupPrefix + ".customFields", errors);
- if (!errors.isEmpty()) {
- throw invalid(errors.size() + (errors.size() == 1 ? " validation error" : " validation errors")
- + " detected: " + String.join("; ", errors));
- }
+ rejectViolations(errors);
if (thing != null) {
thing = validateThing(thing, managedFields);
}
@@ -137,16 +150,168 @@ public class IotFleetIndexingService {
/** The schema of an enabled index. */
public String describeIndex(String indexName, String region) {
+ String schema = schema(indexName, getIndexingConfiguration(region));
+ if (schema == null) {
+ throw new AwsException("ResourceNotFoundException", "Index " + indexName + " does not exist", 404);
+ }
+ return schema;
+ }
+
+ /**
+ * SearchIndex over the things of the caller's account and region, in thing name order. A
+ * request is checked in the order measured on AWS: the body and member types, member
+ * constraints, an empty index name, the query version, an unknown or disabled index, the query,
+ * then the page token. The token names the request it continues (its query string, index name
+ * and version as sent) and the last thing returned, so a page continues after that thing
+ * whatever was deleted before it. It also carries a checksum, so an edited token is invalid.
+ */
+ public IotService.Page<ObjectNode> searchIndex(JsonNode request, String region) {
+ if (!request.isObject()) {
+ throw serializationError("request body", JsonNodeType.OBJECT);
+ }
+ String queryString = text(request.path("queryString"), "queryString");
+ JsonNode maxResults = request.path("maxResults");
+ if (present(maxResults) && !maxResults.isNumber()) {
+ throw serializationError("maxResults", JsonNodeType.NUMBER);
+ }
+ // AWS truncates maxResults toward zero and saturates it at the int range, as this cast does.
+ int pageSize = present(maxResults) ? (int) maxResults.asDouble() : Integer.MAX_VALUE;
+ String requestedIndexName = text(request.path("indexName"), "indexName");
+ String queryVersion = text(request.path("queryVersion"), "queryVersion");
+ String nextToken = text(request.path("nextToken"), "nextToken");
+ List<String> errors = new ArrayList<>();
+ if (queryString == null) {
+ errors.add("Value null at 'queryString' failed to satisfy constraint: Member must not be null");
+ } else if (queryString.isEmpty()) {
+ errors.add("Value '' at 'queryString' failed to satisfy constraint: "
+ + "Member must have length greater than or equal to 1");
+ } else if (queryString.codePointCount(0, queryString.length()) > MAX_QUERY_STRING_LENGTH) {
+ errors.add("Value '" + queryString + "' at 'queryString' failed to satisfy constraint: "
+ + "Member must have length less than or equal to " + MAX_QUERY_STRING_LENGTH);
+ }
+ if (pageSize < 1) {
+ errors.add("Value '" + pageSize + "' at 'maxResults' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 1");
+ }
+ rejectViolations(errors);
+ String indexName = requestedIndexName == null ? THINGS_INDEX : requestedIndexName;
+ if (indexName.isEmpty()) {
+ throw invalid("indexName cannot be empty.");
+ }
+ if (queryVersion != null && !"2017-09-30".equals(queryVersion)) {
+ throw invalid("Invalid queryVersion. Expected one of: [2017-09-30]");
+ }
IotIndexingConfiguration configuration = getIndexingConfiguration(region);
- String schema = switch (indexName) {
+ if (schema(indexName, configuration) == null) {
+ throw new AwsException("ResourceNotFoundException", "Index " + indexName
+ + " does not exist. Please enable index by calling UpdateIndexingConfiguration", 404);
+ }
+ if (THING_GROUPS_INDEX.equals(indexName)) {
+ // ponytail: thing group documents are not modeled; refusing beats an empty result.
+ throw invalid("Floci does not support searching AWS_ThingGroups");
+ }
+ ThingIndexing indexing = configuration.thing();
+ Predicate<JsonNode> query = IotFleetIndexQuery.parse(queryString, indexing);
+ String parameters = pageParameters(requestedIndexName, queryVersion, queryString);
+ String after = pageAfter(nextToken, parameters);
+ Map<String, List<String>> thingGroupNames = thingGroupNames(region);
+ List<ObjectNode> matches = new ArrayList<>();
+ for (Thing thing : iotService.listThings(region)) {
+ String thingName = thing.getThingName();
+ if (after != null && thingName.compareTo(after) <= 0) {
+ continue;
+ }
+ ObjectNode document = document(thing, thingGroupNames.getOrDefault(thingName, List.of()));
+ if (query.test(document)) {
+ matches.add(document);
+ if (matches.size() > pageSize) {
+ break;
+ }
+ }
+ }
+ if (matches.size() <= pageSize) {
+ return new IotService.Page<>(matches, null);
+ }
+ matches.removeLast();
+ return new IotService.Page<>(matches, pageToken(parameters, matches.getLast().path("thingName").asText()));
+ }
+
+ /**
+ * A digest of the members a page token is bound to, each as sent: AWS treats an omitted index
+ * name or version as different from its explicit default.
+ */
+ private static String pageParameters(String indexName, String queryVersion, String queryString) {
+ // The index name and version were validated against closed sets, so only the query can hold a newline.
+ return UUID.nameUUIDFromBytes((indexName + "\n" + queryVersion + "\n" + queryString)
+ .getBytes(StandardCharsets.UTF_8)).toString();
+ }
+
+ /** The thing name a page token continues after, null for the first page. */
+ private static String pageAfter(String nextToken, String parameters) {
+ if (nextToken == null) {
+ return null;
+ }
+ Matcher token;
+ try {
+ token = PAGE_TOKEN.matcher(new String(Base64.getUrlDecoder().decode(nextToken), StandardCharsets.UTF_8));
+ } catch (IllegalArgumentException e) {
+ throw invalid("Invalid nextToken");
+ }
+ if (!token.matches() || !token.group(2).equals(pageCheck(token.group(1), token.group(3)))) {
+ throw invalid("Invalid nextToken");
+ }
+ if (!token.group(1).equals(parameters)) {
+ throw invalid("nextToken was from a request with different parameters");
+ }
+ return token.group(3);
+ }
+
+ private static String pageToken(String parameters, String thingName) {
+ return Base64.getUrlEncoder().withoutPadding().encodeToString(
+ (parameters + pageCheck(parameters, thingName) + thingName).getBytes(StandardCharsets.UTF_8));
+ }
+
+ /** A checksum of a token's parameters and cursor, so an edited or truncated token is invalid, as on AWS. */
+ private static String pageCheck(String parameters, String thingName) {
+ return UUID.nameUUIDFromBytes((parameters + thingName).getBytes(StandardCharsets.UTF_8)).toString();
+ }
+
+ /** The thing groups each thing of the region is a direct member of, in group name order. */
+ private Map<String, List<String>> thingGroupNames(String region) {
+ Map<String, List<String>> groups = new HashMap<>();
+ for (IotThingGroup group : iotService.listThingGroups(region, null, null).items()) {
+ for (String thingName : iotService.listThingsInThingGroup(group.getThingGroupName(), region)) {
+ groups.computeIfAbsent(thingName, ignored -> new ArrayList<>()).add(group.getThingGroupName());
+ }
+ }
+ return groups;
+ }
+
+ /** A thing's index document; like AWS it leaves out a member the thing does not have. */
+ private static ObjectNode document(Thing thing, List<String> thingGroupNames) {
+ ObjectNode document = JsonNodeFactory.instance.objectNode();
+ document.put("thingName", thing.getThingName());
+ document.put("thingId", thing.getThingId());
+ if (thing.getThingTypeName() != null) {
+ document.put("thingTypeName", thing.getThingTypeName());
+ }
+ if (!thingGroupNames.isEmpty()) {
+ thingGroupNames.forEach(document.putArray("thingGroupNames")::add);
+ }
+ if (thing.getAttributes() != null && !thing.getAttributes().isEmpty()) {
+ ObjectNode attributes = document.putObject("attributes");
+ thing.getAttributes().forEach(attributes::put);
+ }
+ return document;
+ }
+
+ /** The schema of the named index, null while it is disabled. */
+ private static String schema(String indexName, IotIndexingConfiguration configuration) {
+ return switch (indexName) {
case THINGS_INDEX -> thingsSchema(configuration.thing());
case THING_GROUPS_INDEX -> "OFF".equals(configuration.thingGroupIndexingMode()) ? null : "REGISTRY";
default -> throw invalid("Unrecognized indexName " + indexName);
};
- if (schema == null) {
- throw new AwsException("ResourceNotFoundException", "Index " + indexName + " does not exist", 404);
- }
- return schema;
}
/** The managed fields AWS derives from the thing indexing modes; none while indexing is OFF. */
@@ -289,6 +454,13 @@ public class IotFleetIndexingService {
return value;
}
+ private static void rejectViolations(List<String> errors) {
+ if (!errors.isEmpty()) {
+ throw invalid(errors.size() + (errors.size() == 1 ? " validation error" : " validation errors")
+ + " detected: " + String.join("; ", errors));
+ }
+ }
+
private static String enumError(String value, String path, List<String> allowed) {
return "Value '" + value + "' at '" + path
+ "' failed to satisfy constraint: Member must satisfy enum value set: " + allowed;
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingIntegrationTest.java
index f75a03282c..790e5c242d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingIntegrationTest.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.iot;
import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.ValidatableResponse;
import org.junit.jupiter.api.Test;
import static io.restassured.RestAssured.given;
@@ -9,17 +10,20 @@ import static org.hamcrest.Matchers.anEmptyMap;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.emptyOrNullString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItems;
import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.startsWith;
/**
* Drives fleet indexing the way the AWS SDKs do: UpdateIndexingConfiguration and
- * GetIndexingConfiguration on {@code /indexing/config}, DescribeIndex on {@code /indices/{indexName}}.
- * Every test signs as its own account, so the configurations they store never meet.
+ * GetIndexingConfiguration on {@code /indexing/config}, DescribeIndex on {@code /indices/{indexName}}
+ * and SearchIndex on {@code /indices/search}. Every test signs as its own account, so the
+ * configurations and things they store never meet.
*/
@QuarkusTest
class IotFleetIndexingIntegrationTest {
@@ -283,4 +287,208 @@ class IotFleetIndexingIntegrationTest {
.statusCode(200)
.body("thingIndexingConfiguration.thingIndexingMode", equalTo("OFF"));
}
+
+ private static void post(String auth, String path, String body) {
+ given()
+ .header("Authorization", auth)
+ .contentType("application/json")
+ .body(body)
+ .when()
+ .post(path)
+ .then()
+ .statusCode(200);
+ }
+
+ private static ValidatableResponse search(String auth, String body) {
+ return given()
+ .header("Authorization", auth)
+ .contentType("application/json")
+ .body(body)
+ .when()
+ .post("/indices/search")
+ .then();
+ }
+
+ @Test
+ void searchIndexReturnsTheSdkWireShape() {
+ String auth = auth("111100000013", "us-east-1");
+ update(auth, """
+ {"thingIndexingConfiguration": {"thingIndexingMode": "REGISTRY", "thingConnectivityIndexingMode": "STATUS"}}
+ """);
+ post(auth, "/thing-types/search-sensor", "{}");
+ post(auth, "/thing-groups/search-north", "{}");
+ post(auth, "/things/search-a", """
+ {"thingTypeName": "search-sensor", "attributePayload": {"attributes": {"provider": "Acme", "site": "north"}}}
+ """);
+ post(auth, "/things/search-b", """
+ {"attributePayload": {"attributes": {"provider": "acme"}}}
+ """);
+ post(auth, "/things/search-c", "{}");
+ given()
+ .header("Authorization", auth)
+ .contentType("application/json")
+ .body("""
+ {"thingGroupName": "search-north", "thingName": "search-a"}
+ """)
+ .when()
+ .put("/thing-groups/addThingToThingGroup")
+ .then()
+ .statusCode(200);
+
+ search(auth, """
+ {"queryString": "attributes.provider:acme AND thingTypeName:search-sensor"}
+ """)
+ .statusCode(200)
+ .body("$", aMapWithSize(1))
+ .body("things", hasSize(1))
+ .body("things[0]", aMapWithSize(5))
+ .body("things[0].thingName", equalTo("search-a"))
+ .body("things[0].thingId", not(emptyOrNullString()))
+ .body("things[0].thingTypeName", equalTo("search-sensor"))
+ .body("things[0].thingGroupNames", contains("search-north"))
+ .body("things[0].attributes.provider", equalTo("Acme"))
+ .body("things[0].attributes.site", equalTo("north"))
+ .body("things[0]", not(hasKey("connectivity")));
+ search(auth, """
+ {"queryString": "thingName:search-c", "queryVersion": "2017-09-30", "indexName": "AWS_Things"}
+ """)
+ .statusCode(200)
+ .body("things[0]", aMapWithSize(2))
+ .body("things[0].thingName", equalTo("search-c"))
+ .body("things[0]", not(hasKey("connectivity")));
+ search(auth, """
+ {"queryString": "connectivity.connected:true"}
+ """)
+ .statusCode(400)
+ .body("__type", equalTo("InvalidQueryException"))
+ .body("message", equalTo("Floci does not support the field connectivity.connected in fleet index "
+ + "queries, query string: connectivity.connected:true"));
+
+ String nextToken = search(auth, """
+ {"queryString": "thingName:search-*", "maxResults": 2}
+ """)
+ .statusCode(200)
+ .body("things.thingName", contains("search-a", "search-b"))
+ .body("nextToken", notNullValue())
+ .extract().path("nextToken");
+ search(auth, "{\"queryString\": \"thingName:search-*\", \"maxResults\": 2, \"nextToken\": \"" + nextToken + "\"}")
+ .statusCode(200)
+ .body("things.thingName", contains("search-c"))
+ .body("$", not(hasKey("nextToken")));
+ }
+
+ @Test
+ void searchIndexRejectsBadRequestsWithAwsErrors() {
+ String auth = auth("111100000008", "us-east-1");
+ update(auth, """
+ {"thingIndexingConfiguration": {"thingIndexingMode": "REGISTRY"}}
+ """);
+
+ search(auth, "{}")
+ .statusCode(400)
+ .body("__type", equalTo("InvalidRequestException"))
+ .body("message", equalTo("1 validation error detected: Value null at 'queryString' failed to satisfy "
+ + "constraint: Member must not be null"));
+ search(auth, """
+ {"queryString": "*", "queryVersion": "2017-09-31"}
+ """)
+ .statusCode(400)
+ .body("__type", equalTo("InvalidRequestException"))
+ .body("message", equalTo("Invalid queryVersion. Expected one of: [2017-09-30]"));
+ search(auth, """
+ {"queryString": "*", "nextToken": "bogus"}
+ """)
+ .statusCode(400)
+ .body("__type", equalTo("InvalidRequestException"))
+ .body("message", equalTo("Invalid nextToken"));
+ search(auth, """
+ {"queryString": "ThingName:x"}
+ """)
+ .statusCode(400)
+ .body("__type", equalTo("InvalidQueryException"))
+ .body("message", equalTo("Unable to parse query, invalid field name, field name: ThingName, "
+ + "query string: ThingName:x"));
+ search(auth, """
+ {"queryString": "thingName>a"}
+ """)
+ .statusCode(400)
+ .body("__type", equalTo("InvalidQueryException"))
+ .body("message", equalTo("Floci does not support comparisons in fleet index queries, "
+ + "query string: thingName>a"));
+ search(auth, """
+ {"queryString": "connectivity.connected:true"}
+ """)
+ .statusCode(400)
+ .body("__type", equalTo("InvalidRequestException"))
+ .body("message", equalTo("Query includes one or more constraints for Connectivity attribute, "
+ + "but Connectivity indexing is not enabled for AWS_Things index"));
+ search(auth, """
+ {"queryString": "*", "maxResults": 9223372036854775808}
+ """)
+ .statusCode(200);
+ search(auth, """
+ {"queryString": "*", "maxResults": -9223372036854775809}
+ """)
+ .statusCode(400)
+ .body("__type", equalTo("InvalidRequestException"))
+ .body("message", equalTo("1 validation error detected: Value '-2147483648' at 'maxResults' failed to "
+ + "satisfy constraint: Member must have value greater than or equal to 1"));
+ }
+
+ @Test
+ void searchIndexOfADisabledIndexIsNotFound() {
+ String auth = auth("111100000009", "us-east-1");
+
+ search(auth, """
+ {"queryString": "*"}
+ """)
+ .statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"))
+ .body("message", equalTo("Index AWS_Things does not exist. "
+ + "Please enable index by calling UpdateIndexingConfiguration"));
+ search(auth, """
+ {"queryString": "*", "indexName": "AWS_ThingGroups"}
+ """)
+ .statusCode(404)
+ .body("__type", equalTo("ResourceNotFoundException"))
+ .body("message", equalTo("Index AWS_ThingGroups does not exist. "
+ + "Please enable index by calling UpdateIndexingConfiguration"));
+
+ update(auth, """
+ {"thingGroupIndexingConfiguration": {"thingGroupIndexingMode": "ON"}}
+ """);
+ search(auth, """
+ {"queryString": "*", "indexName": "AWS_ThingGroups"}
+ """)
+ .statusCode(400)
+ .body("__type", equalTo("InvalidRequestException"))
+ .body("message", equalTo("Floci does not support searching AWS_ThingGroups"));
+ }
+
+ @Test
+ void searchIndexIsIsolatedPerAccountAndRegion() {
+ String home = auth("111100000010", "us-east-1");
+ String otherAccount = auth("111100000011", "us-east-1");
+ String otherRegion = auth("111100000010", "eu-west-1");
+ String indexing = """
+ {"thingIndexingConfiguration": {"thingIndexingMode": "REGISTRY"}}
+ """;
+ update(home, indexing);
+ update(otherAccount, indexing);
+ update(otherRegion, indexing);
+ post(home, "/things/isolated-thing", "{}");
+ String query = """
+ {"queryString": "thingName:isolated-thing"}
+ """;
+
+ search(home, query)
+ .statusCode(200)
+ .body("things.thingName", contains("isolated-thing"));
+ search(otherAccount, query)
+ .statusCode(200)
+ .body("things", empty());
+ search(otherRegion, query)
+ .statusCode(200)
+ .body("things", empty());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingServiceTest.java
index 34699f60f3..2c0dcb9730 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingServiceTest.java
@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.iot;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.services.iot.model.IotIndexingConfiguration;
@@ -14,13 +15,22 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.IntStream;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -62,8 +72,13 @@ class IotFleetIndexingServiceTest {
new Field("thingGroupName", "String"),
new Field("thingGroupId", "String"));
+ private static final String CONNECTIVITY_NOT_ENABLED = "Query includes one or more constraints for Connectivity "
+ + "attribute, but Connectivity indexing is not enabled for AWS_Things index";
+
private final ObjectMapper mapper = new ObjectMapper();
- private final IotFleetIndexingService service = new IotFleetIndexingService(new InMemoryStorage<>());
+ private final IotServiceTestSupport iot = new IotServiceTestSupport(REGION, null);
+ private final IotFleetIndexingService service =
+ new IotFleetIndexingService(new InMemoryStorage<>(), iot.service);
private JsonNode json(String text) {
try {
@@ -675,4 +690,640 @@ class IotFleetIndexingServiceTest {
""");
assertDescribeFails("AWS_Thing", "InvalidRequestException", 400, "Unrecognized indexName AWS_Thing");
}
+
+ /**
+ * Four things in the default region: alpha-1 and beta-1 of type sensor, alpha-1 and alpha-2 in
+ * thing groups, attribute values in mixed case, and gamma with nothing but its name.
+ */
+ private void fleet(String thingIndexing) {
+ update("{\"thingIndexingConfiguration\": " + thingIndexing + "}");
+ IotService registry = iot.service;
+ registry.createThingType("sensor", mapper.createObjectNode(), REGION);
+ registry.createThingGroup("north-group", mapper.createObjectNode(), REGION);
+ registry.createThingGroup("south-group", mapper.createObjectNode(), REGION);
+ registry.createThing("alpha-1", Map.of("site", "North", "provider", "acme"), "sensor", REGION);
+ registry.createThing("alpha-2", Map.of("site", "south", "provider", "ACME"), null, REGION);
+ registry.createThing("beta-1", Map.of("provider", "other"), "sensor", REGION);
+ registry.createThing("gamma", Map.of(), null, REGION);
+ registry.addThingToThingGroup("north-group", "alpha-1", REGION);
+ registry.addThingToThingGroup("south-group", "alpha-2", REGION);
+ registry.addThingToThingGroup("north-group", "alpha-2", REGION);
+ }
+
+ private void fleet() {
+ fleet("{\"thingIndexingMode\": \"REGISTRY\"}");
+ }
+
+ private ObjectNode query(String queryString) {
+ return mapper.createObjectNode().put("queryString", queryString);
+ }
+
+ private IotService.Page<ObjectNode> search(ObjectNode request) {
+ return service.searchIndex(request, REGION);
+ }
+
+ private List<String> names(String queryString) {
+ return search(query(queryString)).items().stream().map(thing -> thing.path("thingName").asText()).toList();
+ }
+
+ private ObjectNode document(String thingName) {
+ List<ObjectNode> things = search(query("thingName:" + thingName)).items();
+ assertEquals(1, things.size());
+ return things.get(0);
+ }
+
+ private void assertSearchFails(ObjectNode request, String code, int status, String message) {
+ AwsException failure = assertThrows(AwsException.class, () -> search(request));
+ assertEquals(code, failure.getErrorCode());
+ assertEquals(status, failure.getHttpStatus());
+ assertEquals(message, failure.getMessage());
+ }
+
+ private void assertQueryFails(String queryString, String message) {
+ assertSearchFails(query(queryString), "InvalidQueryException", 400, message + ", query string: " + queryString);
+ }
+
+ private void assertUnsupported(String queryString, String construct) {
+ assertQueryFails(queryString, "Floci does not support " + construct + " in fleet index queries");
+ }
+
+ @Test
+ void searchMatchesValuesCaseInsensitivelyInThingNameOrder() {
+ fleet();
+
+ assertEquals(List.of("alpha-1"), names("thingName:ALPHA-1"));
+ assertEquals(List.of("alpha-1"), names("attributes.site:north"));
+ assertEquals(List.of("alpha-1", "alpha-2"), names("attributes.provider:Acme"));
+ assertEquals(List.of("alpha-1", "beta-1"), names("thingTypeName:SENSOR"));
+ assertEquals(List.of("alpha-1", "alpha-2"), names("thingGroupNames:north-group"));
+ assertEquals(List.of("alpha-2"), names("thingGroupNames:south-group"));
+ assertEquals(List.of("gamma"), names("thingId:" + iot.service.describeThing("gamma", REGION).getThingId()));
+ assertEquals(List.of("alpha-1"), names("thingName:\"Alpha-1\""));
+ assertEquals(List.of(), names("thingName:alpha"));
+ assertEquals(List.of(), names("attributes.missing:x"));
+ }
+
+ @Test
+ void fieldNamesAreCaseSensitive() {
+ fleet();
+
+ assertEquals(List.of(), names("attributes.Site:north"));
+ assertQueryFails("ThingName:alpha-1", "Unable to parse query, invalid field name, field name: ThingName");
+ assertQueryFails("thingname:alpha-1", "Unable to parse query, invalid field name, field name: thingname");
+ }
+
+ @Test
+ void wildcardsMatchAnyRunOrOneCharacter() {
+ fleet();
+
+ assertEquals(List.of("alpha-1", "alpha-2"), names("thingName:alpha*"));
+ assertEquals(List.of("alpha-1", "alpha-2"), names("thingName:ALPHA-?"));
+ assertEquals(List.of("beta-1"), names("thingName:b?ta-1"));
+ assertEquals(List.of("gamma"), names("thingName:*a"));
+ assertEquals(List.of("alpha-1", "alpha-2"), names("attributes.provider:a*e"));
+ assertEquals(List.of(), names("thingName:alpha?"));
+ assertEquals(List.of(), names("thingName:\"alpha*\""));
+ assertEquals(List.of(), names("thingName:alpha\\*"));
+ }
+
+ @Test
+ void existenceAndMatchAllQueries() {
+ fleet();
+
+ assertEquals(List.of("alpha-1", "beta-1"), names("thingTypeName:*"));
+ assertEquals(List.of("alpha-1", "alpha-2"), names("attributes.site:*"));
+ assertEquals(List.of("alpha-1", "alpha-2"), names("thingGroupNames:*"));
+ assertEquals(List.of("alpha-1", "alpha-2", "beta-1", "gamma"), names("*"));
+ }
+
+ @Test
+ void booleanOperatorsFollowTheMeasuredAwsPrecedence() {
+ fleet();
+
+ // AND binds tighter than OR.
+ assertEquals(List.of("alpha-2", "gamma"),
+ names("thingName:gamma OR attributes.provider:acme AND attributes.site:south"));
+ assertEquals(List.of("alpha-2", "gamma"),
+ names("attributes.provider:acme AND attributes.site:south OR thingName:gamma"));
+ // Whitespace is an AND that binds looser than OR: (gamma OR acme) AND south.
+ assertEquals(List.of("alpha-2"), names("thingName:gamma OR attributes.provider:acme attributes.site:south"));
+ assertEquals(List.of("alpha-1", "alpha-2"), names("thingName:alpha* attributes.provider:acme"));
+ assertEquals(List.of("alpha-2", "gamma"),
+ names("(thingName:gamma OR attributes.provider:acme) AND NOT thingTypeName:sensor"));
+ // NOT applies to the term after it only.
+ assertEquals(List.of("beta-1"), names("NOT attributes.provider:acme AND thingTypeName:sensor"));
+ assertEquals(List.of("alpha-2", "gamma"), names("NOT thingTypeName:sensor"));
+ assertEquals(List.of("alpha-2", "gamma"), names("-thingTypeName:sensor"));
+ assertEquals(List.of("alpha-2"), names("thingName:alpha* -thingTypeName:sensor"));
+ assertEquals(List.of("alpha-2"), names("thingName:alpha* - thingTypeName:sensor"));
+ assertEquals(List.of("alpha-1", "alpha-2", "gamma"), names("-(thingTypeName:sensor AND attributes.provider:other)"));
+ // The symbolic forms AWS accepts.
+ assertEquals(List.of("alpha-2", "gamma"), names("!thingTypeName:sensor"));
+ assertEquals(List.of("alpha-1"), names("thingTypeName:sensor && attributes.site:north"));
+ assertEquals(List.of("beta-1", "gamma"), names("thingName:gamma || attributes.provider:other"));
+ }
+
+ @Test
+ void documentsCarryOnlyTheMembersAThingHas() {
+ fleet();
+
+ assertEquals(json("{\"thingName\": \"alpha-1\", \"thingId\": \""
+ + iot.service.describeThing("alpha-1", REGION).getThingId() + "\", \"thingTypeName\": \"sensor\", "
+ + "\"thingGroupNames\": [\"north-group\"], \"attributes\": {\"site\": \"North\", \"provider\": \"acme\"}}"),
+ document("alpha-1"));
+ assertEquals(json("[\"north-group\", \"south-group\"]"), document("alpha-2").get("thingGroupNames"));
+ assertEquals(json("{\"thingName\": \"gamma\", \"thingId\": \""
+ + iot.service.describeThing("gamma", REGION).getThingId() + "\"}"), document("gamma"));
+ }
+
+ @Test
+ void connectivityFieldsAreRefusedLikeShadowAndDeviceDefenderFields() {
+ fleet("{\"thingIndexingMode\": \"REGISTRY\", \"thingConnectivityIndexingMode\": \"STATUS\"}");
+
+ assertUnsupported("connectivity.connected:true", "the field connectivity.connected");
+ assertUnsupported("connectivity.clientId:alpha-1", "the field connectivity.clientId");
+ assertUnsupported("connectivity.disconnectReason:CONNECTION_LOST", "the field connectivity.disconnectReason");
+ assertUnsupported("thingName:gamma OR connectivity.connected:false", "the field connectivity.connected");
+ }
+
+ @Test
+ void documentsCarryNoConnectivityWhileConnectivityIsIndexed() {
+ fleet("{\"thingIndexingMode\": \"REGISTRY\", \"thingConnectivityIndexingMode\": \"STATUS\"}");
+
+ assertFalse(document("alpha-1").has("connectivity"));
+ assertEquals(json("{\"thingName\": \"gamma\", \"thingId\": \""
+ + iot.service.describeThing("gamma", REGION).getThingId() + "\"}"), document("gamma"));
+ }
+
+ @Test
+ void connectivityQueriesNeedConnectivityIndexing() {
+ fleet();
+
+ assertSearchFails(query("connectivity.connected:true"), "InvalidRequestException", 400, CONNECTIVITY_NOT_ENABLED);
+ assertSearchFails(query("thingName:gamma OR connectivity.clientId:gamma"), "InvalidRequestException", 400,
+ CONNECTIVITY_NOT_ENABLED);
+ assertFalse(document("alpha-1").has("connectivity"));
+ }
+
+ @Test
+ void shadowAndDeviceDefenderQueriesNeedTheirIndexing() {
+ fleet();
+
+ assertSearchFails(query("shadow.reported.x:1"), "InvalidRequestException", 400, "Query includes one or more "
+ + "constraints for Shadow attribute, but Shadow indexing is not enabled for AWS_Things index");
+ assertSearchFails(query("deviceDefender.violationCount:1"), "InvalidRequestException", 400,
+ "Query includes one or more constraints for Devicedefender attribute, but Devicedefender indexing "
+ + "is not enabled for AWS_Things index");
+ }
+
+ @Test
+ void classicAndNamedShadowFieldsFollowTheirOwnIndexing() {
+ fleet("""
+ {"thingIndexingMode": "REGISTRY", "namedShadowIndexingMode": "ON",
+ "filter": {"namedShadowNames": ["building"]}}
+ """);
+
+ assertSearchFails(query("shadow.reported.x:1"), "InvalidRequestException", 400, "Query includes one or more "
+ + "constraints for Shadow attribute, but Shadow indexing is not enabled for AWS_Things index");
+ assertUnsupported("shadow.name.building.reported.type:x", "the field shadow.name.building.reported.type");
+ assertQueryFails("shadow.name.other.reported.a:1",
+ "Unable to parse query, invalid field name, field name: shadow.name.other.reported.a");
+ assertQueryFails("shadow.name.building:x",
+ "Unable to parse query, invalid field name, field name: shadow.name.building");
+ assertQueryFails("shadow.name:x", "Unable to parse query, invalid field name, field name: shadow.name");
+
+ update("""
+ {"thingIndexingConfiguration": {"thingIndexingMode": "REGISTRY_AND_SHADOW"}}
+ """);
+ assertQueryFails("shadow.name.building.reported.type:x",
+ "Unable to parse query, invalid field name, field name: shadow.name.building.reported.type");
+ assertUnsupported("shadow.reported.x:1", "the field shadow.reported.x");
+ }
+
+ @Test
+ void awsValidConstructsFlociDoesNotEvaluateAreRefused() {
+ fleet("""
+ {"thingIndexingMode": "REGISTRY_AND_SHADOW", "thingConnectivityIndexingMode": "STATUS",
+ "deviceDefenderIndexingMode": "VIOLATIONS"}
+ """);
+
+ assertUnsupported("attributes.site:[a TO z]", "range queries");
+ assertUnsupported("attributes.site:{a TO z}", "range queries");
+ assertUnsupported("thingName>a", "comparisons");
+ assertUnsupported("attributes.count>=5", "comparisons");
+ assertUnsupported("attributes.count<5", "comparisons");
+ assertUnsupported("attributes.count<=5", "comparisons");
+ assertUnsupported("alpha", "free text terms");
+ assertUnsupported("thingName:alpha-1 or thingName:gamma", "free text terms");
+ assertUnsupported("\"alpha-1\"", "free text terms");
+ assertUnsupported("attributes.site:(north OR south)", "field grouping");
+ assertUnsupported("shadow.reported.x:1", "the field shadow.reported.x");
+ assertUnsupported("deviceDefender.violationCount:1", "the field deviceDefender.violationCount");
+ assertUnsupported("connectivity.timestamp:0", "the field connectivity.timestamp");
+ assertUnsupported("connectivity.connected:true", "the field connectivity.connected");
+ }
+
+ @Test
+ void malformedQueriesAreSyntaxErrors() {
+ fleet();
+
+ for (String queryString : List.of("thingName:", "thingName:(a", "thingName:a AND", "AND thingName:a",
+ "thingName:a OR", "thingName:a NOT", "NOT", "!", "()", "thingName:a)", "thingName:a -",
+ "NOT NOT thingName:a", "NOT -thingName:a", "-NOT thingName:a", "thingName:a AND OR thingName:b",
+ "thingName:\"a", "thingName:a\"", " ", "attributes.count:>5")) {
+ assertQueryFails(queryString, "Unable to parse query, invalid syntax");
+ }
+ }
+
+ @Test
+ void rangesAreParsedAsAwsParsesThem() {
+ fleet();
+
+ for (String queryString : List.of("thingName:[", "thingName:[a", "thingName:[a TO", "thingName:[a TO b",
+ "thingName:[a to b]", "thingName:[a TO b TO c]", "thingName:{", "thingName:]", "thingName:[]",
+ "thingName:a]", "thingName:[aTOb]", "thingName:[a TO b]]", "thingName:a]b", "thingName:a[b",
+ "thingName:a}", "thingName:a{b", "attributes.a[0]:x", "thingName:[TO b]", "thingName:[a TO]",
+ "[a TO]", "thingName:[a TO b)", "thingName:[a\tTO\tb]")) {
+ assertQueryFails(queryString, "Unable to parse query, invalid syntax");
+ }
+ assertQueryFails("bogusfield:1 OR thingName:[a", "Unable to parse query, invalid syntax");
+ for (String queryString : List.of("thingName:[a b]", "thingName:[a TO b}", "thingName:[* TO b]",
+ "thingName:[ a TO b ]", "thingName:[\"a\" TO \"b\"]", "thingName:[\"a]\" TO b]",
+ "thingName:[a[ TO b]", "NOT thingName:[a TO b]", "(thingName:[a TO b])",
+ "thingName:[a TO b](thingName:c)")) {
+ assertUnsupported(queryString, "range queries");
+ }
+ assertUnsupported("[a TO b]", "free text terms");
+ assertEquals(List.of(), names("thingName:a\\]"));
+ assertEquals(List.of(), names("thingName:\"a]\""));
+
+ String range = "thingName:[a TO b] ";
+ String eight = String.join(" ", Collections.nCopies(8, "thingName:x"));
+ assertUnsupported(range.repeat(4) + eight, "range queries");
+ assertQueryFails(range.repeat(5) + eight,
+ "Unable to parse query, number of terms cannot exceed 12, terms found: 13");
+ }
+
+ @Test
+ void unknownFieldsAreInvalidFieldNames() {
+ fleet();
+
+ for (String field : List.of("bogusfield", "registry.version", "*", "attributes", "attributes.")) {
+ assertQueryFails(field + ":1", "Unable to parse query, invalid field name, field name: " + field);
+ }
+ }
+
+ @Test
+ void queryTypesAwsRejectsAreRejectedWithItsMessages() {
+ fleet();
+
+ assertQueryFails("+thingName:a", "Unable to parse query, unsupported operator - \"+\"");
+ assertQueryFails("thingName:a~", "Unable to parse query, unsupported query type - fuzzy");
+ assertQueryFails("thingName:a~2", "Unable to parse query, unsupported query type - fuzzy");
+ assertQueryFails("thingName:/a/", "Unable to parse query, unsupported query type - regular expression");
+ assertQueryFails("thingName:a^2", "Unable to parse query, unsupported query type - boost");
+ }
+
+ @Test
+ void escapedFuzzyAndBoostCharactersAreLiteral() {
+ fleet();
+
+ assertEquals(List.of(), names("thingName:a\\~b"));
+ assertEquals(List.of(), names("attributes.x:a\\^b"));
+ assertEquals(List.of(), names("thingName:\"a~b^c\""));
+ assertQueryFails("thingName:a~", "Unable to parse query, unsupported query type - fuzzy");
+ }
+
+ @Test
+ void aQueryStringOfAtMost1000CodePointsIsAccepted() {
+ fleet();
+ String longest = "thingName:" + "\u00e9".repeat(990);
+ String tooLong = longest + "\u00e9";
+
+ assertEquals(List.of(), names(longest));
+ assertSearchFails(query(tooLong), "InvalidRequestException", 400, "1 validation error detected: Value '"
+ + tooLong + "' at 'queryString' failed to satisfy constraint: "
+ + "Member must have length less than or equal to 1000");
+ assertEquals(List.of("gamma"), names("(".repeat(490) + "thingName:gamma" + ")".repeat(490)));
+ }
+
+ @Test
+ void aQueryOfAtMost12TermsIsAccepted() {
+ fleet();
+ String twelve = String.join(" OR ", Collections.nCopies(12, "thingName:gamma"));
+ String tooMany = "Unable to parse query, number of terms cannot exceed 12, terms found: 13";
+
+ assertEquals(List.of("gamma"), names(twelve));
+ assertEquals(List.of("alpha-1", "alpha-2", "beta-1", "gamma"),
+ names(String.join(" OR ", Collections.nCopies(7, "NOT thingName:z"))));
+ assertQueryFails(twelve + " OR thingName:gamma", tooMany);
+ assertQueryFails("* " + twelve, tooMany);
+ assertQueryFails("(" + twelve + ") AND NOT thingName:z", tooMany);
+ assertQueryFails("bogusfield:1 OR " + twelve, tooMany);
+ assertQueryFails(twelve + " OR thingName:*a*b*c*", tooMany);
+ }
+
+ @Test
+ void syntaxErrorsWinOverTheTermLimitAndOtherQueryErrors() {
+ fleet();
+
+ for (String queryString : List.of("thingName:z OR ".repeat(12) + "thingName:", "bogusfield:1 OR thingName:",
+ "thingName:a~ OR thingName:", "thingName:*a*b*c* OR thingName:",
+ "attributes.site:[a TO b] OR thingName:", "+thingName:a OR thingName:", "thingName:?a OR thingName:",
+ "bogusfield:1 OR thingName:<a", "bogusfield:1 OR (thingName:a", "bogusfield:1 AND", "bogusfield:",
+ "deviceDefender.violationCount:1 OR thingName:")) {
+ assertQueryFails(queryString, "Unable to parse query, invalid syntax");
+ }
+ }
+
+ @Test
+ void theTermLimitWinsOverAFieldWhoseIndexingIsOff() {
+ fleet();
+ List<String> twelve = IntStream.rangeClosed(1, 12).mapToObj(i -> "thingName:z" + i).toList();
+
+ assertQueryFails("deviceDefender.violationCount:1 OR " + String.join(" OR ", twelve) + " OR thingName:y",
+ "Unable to parse query, number of terms cannot exceed 12, terms found: 14");
+ }
+
+ @Test
+ void fieldGroupValuesCountAsTermsButTheFieldDoesNot() {
+ fleet();
+ List<String> values = List.of("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m");
+ String tooMany = "Unable to parse query, number of terms cannot exceed 12, terms found: 13";
+
+ assertQueryFails("thingName:(" + String.join(" OR ", values) + ")", tooMany);
+ assertQueryFails("thingName:(" + String.join(" OR ", values.subList(0, 12)) + ") OR thingName:x", tooMany);
+ assertUnsupported("thingName:(" + String.join(" OR ", values.subList(0, 11)) + ") OR thingName:x",
+ "field grouping");
+ for (String queryString : List.of("thingName:()", "thingName:(a OR)", "thingName:(a OR b")) {
+ assertQueryFails(queryString, "Unable to parse query, invalid syntax");
+ }
+ }
+
+ @Test
+ void aFieldGroupChecksItsFieldFirst() {
+ fleet();
+ String invalidFieldName = "Unable to parse query, invalid field name, field name: ";
+
+ assertQueryFails("bogusfield:(a OR b)", invalidFieldName + "bogusfield");
+ assertQueryFails("registry.version:(1 OR 2)", invalidFieldName + "registry.version");
+ assertQueryFails("attributes:(a OR b)", invalidFieldName + "attributes");
+ assertQueryFails("bogusfield:(a OR b) OR deviceDefender.violationCount:1", invalidFieldName + "bogusfield");
+ assertQueryFails("attributes.a[0]:(x OR y)", "Unable to parse query, invalid syntax");
+ assertQueryFails("bogusfield:(a OR b) OR thingName:", "Unable to parse query, invalid syntax");
+ assertQueryFails("bogusfield:(a OR b OR c OR d OR e OR f OR g OR h OR i OR j OR k OR l OR m)",
+ "Unable to parse query, number of terms cannot exceed 12, terms found: 13");
+ assertSearchFails(query("connectivity.connected:(true OR false)"), "InvalidRequestException", 400,
+ CONNECTIVITY_NOT_ENABLED);
+ assertSearchFails(query("deviceDefender.violationCount:(1 OR 2)"), "InvalidRequestException", 400,
+ "Query includes one or more constraints for Devicedefender attribute, but Devicedefender indexing "
+ + "is not enabled for AWS_Things index");
+ assertUnsupported("thingName:(a OR b)", "field grouping");
+ }
+
+ @Test
+ void aValueTakesAtMostTwoUnescapedWildcards() {
+ fleet();
+
+ assertQueryFails("thingName:*a*b*", "Unable to parse query, invalid value - too many wildcards (*). "
+ + "The maximum allowed per term is 2, field name: thingName, value: *a*b*");
+ assertQueryFails("attributes.x:*a*b*c", "Unable to parse query, invalid value - too many wildcards (*). "
+ + "The maximum allowed per term is 2, field name: attributes.x, value: *a*b*c");
+ assertEquals(List.of("alpha-1", "alpha-2", "beta-1", "gamma"), names("thingName:*a*"));
+ assertEquals(List.of(), names("thingName:*a\\*b*"));
+ assertEquals(List.of(), names("thingName:\"*a*b*c*\""));
+ assertEquals(List.of(), names("thingName:a\\*b\\*c\\*d"));
+ }
+
+ @Test
+ void aValueCannotStartWithAQuestionMark() {
+ fleet();
+
+ assertQueryFails("thingName:?a", "Unable to parse query, invalid value - wildcard character (?) is not "
+ + "allowed at the start of the value, field name: thingName, value: ?a");
+ assertEquals(List.of(), names("thingName:a?b"));
+ assertEquals(List.of(), names("thingName:a?b?c?d?e?f"));
+ }
+
+ @Test
+ void requestsAreValidatedInTheMeasuredAwsOrder() {
+ assertSearchFails(mapper.createObjectNode(), "InvalidRequestException", 400, "1 validation error detected: "
+ + "Value null at 'queryString' failed to satisfy constraint: Member must not be null");
+ assertSearchFails(query(""), "InvalidRequestException", 400, "1 validation error detected: Value '' at "
+ + "'queryString' failed to satisfy constraint: Member must have length greater than or equal to 1");
+ assertSearchFails(query("*").put("maxResults", 0), "InvalidRequestException", 400,
+ "1 validation error detected: Value '0' at 'maxResults' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 1");
+ assertSearchFails(query("*").put("indexName", ""), "InvalidRequestException", 400,
+ "indexName cannot be empty.");
+ assertSearchFails(query("*").put("indexName", "Nope").put("queryVersion", "bogus"), "InvalidRequestException",
+ 400, "Invalid queryVersion. Expected one of: [2017-09-30]");
+ assertSearchFails(query("*").put("indexName", "Nope"), "InvalidRequestException", 400,
+ "Unrecognized indexName Nope");
+ assertSearchFails(query("bogusfield:1"), "ResourceNotFoundException", 404,
+ "Index AWS_Things does not exist. Please enable index by calling UpdateIndexingConfiguration");
+ assertSearchFails(query("*").put("indexName", "AWS_ThingGroups"), "ResourceNotFoundException", 404,
+ "Index AWS_ThingGroups does not exist. Please enable index by calling UpdateIndexingConfiguration");
+
+ update("""
+ {"thingIndexingConfiguration": {"thingIndexingMode": "REGISTRY"},
+ "thingGroupIndexingConfiguration": {"thingGroupIndexingMode": "ON"}}
+ """);
+ assertSearchFails(query("*").put("indexName", "AWS_ThingGroups"), "InvalidRequestException", 400,
+ "Floci does not support searching AWS_ThingGroups");
+ assertEquals(List.of(), search(query("*").put("indexName", "AWS_Things").put("queryVersion", "2017-09-30"))
+ .items());
+ }
+
+ @Test
+ void pagesCarryATokenUntilTheLastPage() {
+ fleet();
+
+ IotService.Page<ObjectNode> first = search(query("thingName:alpha*").put("maxResults", 1));
+ assertEquals("alpha-1", first.items().get(0).path("thingName").asText());
+ assertEquals(1, first.items().size());
+ assertNotNull(first.nextToken());
+ IotService.Page<ObjectNode> second =
+ search(query("thingName:alpha*").put("maxResults", 1).put("nextToken", first.nextToken()));
+ assertEquals("alpha-2", second.items().get(0).path("thingName").asText());
+ assertNull(second.nextToken());
+ assertNull(search(query("thingName:alpha*")).nextToken());
+
+ assertSearchFails(query("*").put("nextToken", "bogus"), "InvalidRequestException", 400, "Invalid nextToken");
+ assertSearchFails(query("bogusfield:1").put("nextToken", "bogus"), "InvalidQueryException", 400,
+ "Unable to parse query, invalid field name, field name: bogusfield, query string: bogusfield:1");
+ }
+
+ @Test
+ void aHugeMaxResultsReturnsTheRemainingThingsWithoutAToken() {
+ fleet();
+ String token = search(query("*").put("maxResults", 1)).nextToken();
+
+ IotService.Page<ObjectNode> rest =
+ search(query("*").put("maxResults", Integer.MAX_VALUE).put("nextToken", token));
+
+ assertEquals(List.of("alpha-2", "beta-1", "gamma"), thingNames(rest));
+ assertNull(rest.nextToken());
+ }
+
+ @Test
+ void aPageTokenContinuesAfterTheLastThingReturnedWhenEarlierThingsAreDeleted() {
+ fleet();
+ IotService.Page<ObjectNode> first = search(query("thingName:alpha*").put("maxResults", 1));
+ assertEquals(List.of("alpha-1"), thingNames(first));
+ assertNotNull(first.nextToken());
+
+ iot.service.deleteThing("alpha-1", REGION);
+ IotService.Page<ObjectNode> second =
+ search(query("thingName:alpha*").put("maxResults", 1).put("nextToken", first.nextToken()));
+
+ assertEquals(List.of("alpha-2"), thingNames(second));
+ assertNull(second.nextToken());
+ }
+
+ @Test
+ void aPageTokenOnlyContinuesTheRequestThatIssuedIt() {
+ fleet();
+ String token = search(query("thingName:alpha*").put("maxResults", 1)).nextToken();
+
+ assertEquals(List.of("alpha-2"),
+ thingNames(search(query("thingName:alpha*").put("maxResults", 5).put("nextToken", token))));
+ assertEquals(List.of("alpha-2"), thingNames(search(query("thingName:alpha*").put("nextToken", token))));
+ String differentParameters = "nextToken was from a request with different parameters";
+ assertSearchFails(query("thingName:alpha* ").put("nextToken", token), "InvalidRequestException", 400,
+ differentParameters);
+ assertSearchFails(query("*").put("nextToken", token), "InvalidRequestException", 400, differentParameters);
+ assertSearchFails(query("thingName:alpha*").put("indexName", "AWS_Things").put("nextToken", token),
+ "InvalidRequestException", 400, differentParameters);
+ assertSearchFails(query("thingName:alpha*").put("queryVersion", "2017-09-30").put("nextToken", token),
+ "InvalidRequestException", 400, differentParameters);
+
+ String explicit = search(query("thingName:alpha*").put("indexName", "AWS_Things")
+ .put("queryVersion", "2017-09-30").put("maxResults", 1)).nextToken();
+ assertEquals(List.of("alpha-2"), thingNames(search(query("thingName:alpha*").put("indexName", "AWS_Things")
+ .put("queryVersion", "2017-09-30").put("nextToken", explicit))));
+ assertSearchFails(query("thingName:alpha*").put("nextToken", explicit), "InvalidRequestException", 400,
+ differentParameters);
+
+ assertSearchFails(query("bogusfield:1").put("nextToken", token), "InvalidQueryException", 400,
+ "Unable to parse query, invalid field name, field name: bogusfield, query string: bogusfield:1");
+ }
+
+ @Test
+ void aTokenThisOperationCouldNotHaveIssuedIsInvalid() {
+ fleet();
+ String bareThingName = Base64.getUrlEncoder().withoutPadding()
+ .encodeToString("alpha-1".getBytes(StandardCharsets.UTF_8));
+
+ for (String token : List.of("YQ", "", "AAAA", bareThingName)) {
+ assertSearchFails(query("*").put("nextToken", token), "InvalidRequestException", 400,
+ "Invalid nextToken");
+ }
+
+ String issued = search(query("thingName:alpha*").put("maxResults", 1)).nextToken();
+ String decoded = new String(Base64.getUrlDecoder().decode(issued), StandardCharsets.UTF_8);
+ String edited = Base64.getUrlEncoder().withoutPadding()
+ .encodeToString((decoded.substring(0, decoded.length() - 1) + "0").getBytes(StandardCharsets.UTF_8));
+ String shortened = Base64.getUrlEncoder().withoutPadding()
+ .encodeToString(decoded.substring(0, decoded.length() - 1).getBytes(StandardCharsets.UTF_8));
+ String truncated = issued.substring(0, issued.length() - 2);
+ for (String token : List.of(edited, shortened, truncated)) {
+ assertSearchFails(query("thingName:alpha*").put("nextToken", token), "InvalidRequestException", 400,
+ "Invalid nextToken");
+ }
+ assertEquals(List.of("alpha-2"), thingNames(search(query("thingName:alpha*").put("nextToken", issued))));
+ }
+
+ @Test
+ void maxResultsIsAnyJsonNumberOfAtLeastOne() {
+ fleet();
+
+ assertEquals(List.of("alpha-1"), thingNames(search(query("*").put("maxResults", 1.5))));
+ IotService.Page<ObjectNode> all = search(query("*").put("maxResults", 2147483648L));
+ assertEquals(List.of("alpha-1", "alpha-2", "beta-1", "gamma"), thingNames(all));
+ assertNull(all.nextToken());
+ assertEquals(4, search(query("*").put("maxResults", 500)).items().size());
+ assertSearchFails(query("*").put("maxResults", "5"), "SerializationException", 400,
+ "maxResults must be a JSON number");
+ assertSearchFails(query("*").put("maxResults", true), "SerializationException", 400,
+ "maxResults must be a JSON number");
+ }
+
+ @Test
+ void maxResultsIsTruncatedAndSaturatedAtTheIntRange() {
+ fleet();
+
+ IotService.Page<ObjectNode> all = search(query("*").put("maxResults", new BigInteger("9223372036854775808")));
+ assertEquals(List.of("alpha-1", "alpha-2", "beta-1", "gamma"), thingNames(all));
+ assertNull(all.nextToken());
+ assertEquals(4, search(query("*").put("maxResults", new BigDecimal("1e30"))).items().size());
+ assertSearchFails(query("*").put("maxResults", new BigInteger("-9223372036854775809")),
+ "InvalidRequestException", 400, "1 validation error detected: Value '-2147483648' at 'maxResults' "
+ + "failed to satisfy constraint: Member must have value greater than or equal to 1");
+ assertSearchFails(query("*").put("maxResults", 0.5), "InvalidRequestException", 400,
+ "1 validation error detected: Value '0' at 'maxResults' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 1");
+ assertSearchFails(query("*").put("maxResults", -1), "InvalidRequestException", 400,
+ "1 validation error detected: Value '-1' at 'maxResults' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 1");
+ }
+
+ @Test
+ void indexNameMustBeAJsonString() {
+ fleet();
+
+ assertSearchFails(query("*").put("indexName", 1), "SerializationException", 400,
+ "indexName must be a JSON string");
+ assertSearchFails(query("*").put("indexName", true), "SerializationException", 400,
+ "indexName must be a JSON string");
+ }
+
+ @Test
+ void aBodyThatIsNotAJsonObjectIsASerializationError() {
+ fleet();
+
+ for (String body : List.of("[]", "\"x\"", "1", "null", "[{\"queryString\":\"*\"}]")) {
+ assertSearchBodyFails(body, "request body must be a JSON object");
+ }
+ }
+
+ @Test
+ void memberTypeErrorsComeBeforeEveryOtherError() {
+ fleet();
+
+ assertSearchBodyFails("{\"queryString\":\"bogusfield:\",\"nextToken\":1}", "nextToken must be a JSON string");
+ assertSearchBodyFails("{\"queryString\":\"*\",\"indexName\":\"\",\"queryVersion\":1}",
+ "queryVersion must be a JSON string");
+ assertSearchBodyFails("{\"queryString\":\"\",\"nextToken\":true}", "nextToken must be a JSON string");
+ assertSearchBodyFails("{\"queryString\":\"*\",\"indexName\":\"nope\",\"nextToken\":1}",
+ "nextToken must be a JSON string");
+ }
+
+ private void assertSearchBodyFails(String body, String message) {
+ AwsException failure = assertThrows(AwsException.class,
+ () -> service.searchIndex(json(body), REGION), body);
+ assertEquals("SerializationException", failure.getErrorCode(), body);
+ assertEquals(400, failure.getHttpStatus(), body);
+ assertEquals(message, failure.getMessage(), body);
+ }
+
+ private static List<String> thingNames(IotService.Page<ObjectNode> page) {
+ return page.items().stream().map(thing -> thing.path("thingName").asText()).toList();
+ }
+
+ @Test
+ void searchSeesOnlyTheCallersRegion() {
+ fleet();
+ iot.service.createThing("alpha-9", Map.of("provider", "acme"), null, "eu-west-1");
+
+ assertEquals(List.of("alpha-1", "alpha-2"), names("attributes.provider:acme"));
+ assertThrows(AwsException.class, () -> service.searchIndex(query("*"), "eu-west-1"));
+
+ service.updateIndexingConfiguration(json("""
+ {"thingIndexingConfiguration": {"thingIndexingMode": "REGISTRY"}}
+ """), "eu-west-1");
+ List<ObjectNode> other = service.searchIndex(query("attributes.provider:acme"), "eu-west-1").items();
+ assertEquals(1, other.size());
+ assertEquals("alpha-9", other.get(0).path("thingName").asText());
+ }
}