Light, fluffy, and always free - The AWS Local Emulator alternative
feat(iot): search the supported fleet index subset (#4913)
Mario Toffia1082941bc20d
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/IotTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/IotTest.javaindex 2ede66e98f..bdc0d4a57e 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/IotTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/IotTest.java@@ -63,6 +63,7 @@ import software.amazon.awssdk.services.iot.model.GetPolicyRequest;import software.amazon.awssdk.services.iot.model.GetPolicyResponse;import software.amazon.awssdk.services.iot.model.GetTopicRuleRequest;import software.amazon.awssdk.services.iot.model.GetTopicRuleResponse;+import software.amazon.awssdk.services.iot.model.IndexNotReadyException;import software.amazon.awssdk.services.iot.model.IndexStatus;import software.amazon.awssdk.services.iot.model.InvalidRequestException;import software.amazon.awssdk.services.iot.model.ListCertificatesRequest;@@ -88,17 +89,21 @@ import software.amazon.awssdk.services.iot.model.NamedShadowIndexingMode;import software.amazon.awssdk.services.iot.model.RemoveThingFromThingGroupRequest;import software.amazon.awssdk.services.iot.model.ResourceAlreadyExistsException;import software.amazon.awssdk.services.iot.model.ResourceNotFoundException;+import software.amazon.awssdk.services.iot.model.SearchIndexRequest;+import software.amazon.awssdk.services.iot.model.SearchIndexResponse;import software.amazon.awssdk.services.iot.model.ServerCertificateStatus;import software.amazon.awssdk.services.iot.model.ServiceType;import software.amazon.awssdk.services.iot.model.SqsAction;import software.amazon.awssdk.services.iot.model.Tag;import software.amazon.awssdk.services.iot.model.TagResourceRequest;import software.amazon.awssdk.services.iot.model.ThingConnectivityIndexingMode;+import software.amazon.awssdk.services.iot.model.ThingDocument;import software.amazon.awssdk.services.iot.model.ThingGroupIndexingConfiguration;import software.amazon.awssdk.services.iot.model.ThingGroupIndexingMode;import software.amazon.awssdk.services.iot.model.ThingGroupProperties;import software.amazon.awssdk.services.iot.model.ThingIndexingConfiguration;import software.amazon.awssdk.services.iot.model.ThingIndexingMode;+import software.amazon.awssdk.services.iot.model.ThingTypeDefinition;import software.amazon.awssdk.services.iot.model.ThingTypeProperties;import software.amazon.awssdk.services.iot.model.TopicRulePayload;import software.amazon.awssdk.services.iot.model.UntagResourceRequest;@@ -140,8 +145,14 @@ import java.io.OutputStream;import java.net.InetSocketAddress;import java.net.Socket;import java.nio.charset.StandardCharsets;+import java.time.Instant;+import java.util.ArrayList;import java.util.Arrays;+import java.util.Comparator;+import java.util.List;import java.util.Map;+import java.util.Set;+import java.util.TreeSet;import static org.assertj.core.api.Assertions.assertThat;import static org.assertj.core.api.Assertions.assertThatThrownBy;@@ -359,6 +370,173 @@ class IotTest {}}+ @Test+ void fleetIndexSearch() throws InterruptedException {+ String prefix = "fleetsearch" + Long.toString(System.currentTimeMillis(), 36);+ String group = prefix + "group";+ String provider = prefix + "p";+ // AWS keeps a deprecated thing type for five minutes before it can be deleted, so on AWS the+ // test borrows a test thing type the account already has rather than creating one.+ String thingType = TestFixtures.isRealAws() ? existingTestThingType() : prefix + "type";+ GetIndexingConfigurationResponse prior =+ iot.getIndexingConfiguration(GetIndexingConfigurationRequest.builder().build());+ try {+ iot.updateIndexingConfiguration(UpdateIndexingConfigurationRequest.builder()+ .thingIndexingConfiguration(ThingIndexingConfiguration.builder()+ .thingIndexingMode(ThingIndexingMode.REGISTRY)+ .thingConnectivityIndexingMode(ThingConnectivityIndexingMode.STATUS)+ .build())+ .build());+ if (!TestFixtures.isRealAws()) {+ iot.createThingType(CreateThingTypeRequest.builder().thingTypeName(thingType).build());+ }+ iot.createThingGroup(CreateThingGroupRequest.builder().thingGroupName(group).build());+ iot.createThing(CreateThingRequest.builder()+ .thingName(prefix + "a")+ .thingTypeName(thingType)+ .attributePayload(AttributePayload.builder()+ .attributes(Map.of("provider", provider + "1", "site", "north"))+ .build())+ .build());+ iot.createThing(CreateThingRequest.builder()+ .thingName(prefix + "b")+ .attributePayload(AttributePayload.builder().attributes(Map.of("provider", provider + "1")).build())+ .build());+ iot.createThing(CreateThingRequest.builder()+ .thingName(prefix + "c")+ .attributePayload(AttributePayload.builder().attributes(Map.of("provider", provider + "2")).build())+ .build());+ iot.addThingToThingGroup(AddThingToThingGroupRequest.builder()+ .thingGroupName(group)+ .thingName(prefix + "a")+ .build());++ List<ThingDocument> all = awaitSearch("thingName:" + prefix + "*", prefix, "a", "b", "c");+ ThingDocument typed = all.get(0);+ assertThat(typed.thingTypeName()).isEqualTo(thingType);+ assertThat(typed.thingGroupNames()).containsExactly(group);+ assertThat(typed.attributes()).containsOnly(Map.entry("provider", provider + "1"), Map.entry("site", "north"));+ assertThat(typed.shadow()).isNull();+ ThingDocument plain = all.get(1);+ assertThat(plain.thingTypeName()).isNull();+ assertThat(plain.hasThingGroupNames()).isFalse();+ assertThat(plain.attributes()).containsOnly(Map.entry("provider", provider + "1"));+ for (ThingDocument thing : all) {+ assertThat(thing.thingId()).isNotBlank();+ }++ awaitSearch("attributes.provider:" + provider + "1", prefix, "a", "b");+ awaitSearch("thingName:" + prefix + "c", prefix, "c");+ if (thingType != null) {+ awaitSearch("thingTypeName:" + thingType, prefix, "a");+ awaitSearch("thingTypeName:" + thingType + " AND attributes.provider:" + provider + "1", prefix, "a");+ }+ awaitSearch("attributes.provider:" + provider + "2 OR thingName:" + prefix + "a", prefix, "a", "c");+ awaitSearch("thingName:" + prefix + "* AND NOT attributes.provider:" + provider + "1", prefix, "c");+ awaitSearch("attributes.provider:" + provider + "?", prefix, "a", "b", "c");+ awaitSearch("thingGroupNames:" + group, prefix, "a");++ Set<String> paged = new TreeSet<>();+ String nextToken = null;+ int pages = 0;+ do {+ SearchIndexResponse page = iot.searchIndex(SearchIndexRequest.builder()+ .queryString("thingName:" + prefix + "*")+ .maxResults(1)+ .nextToken(nextToken)+ .build());+ assertThat(page.things()).hasSizeLessThanOrEqualTo(1);+ page.things().forEach(thing -> paged.add(thing.thingName()));+ nextToken = page.nextToken();+ pages++;+ } while (nextToken != null && pages < 10);+ assertThat(paged).containsExactly(prefix + "a", prefix + "b", prefix + "c");+ assertThat(pages).isBetween(3, 4);++ assertThatThrownBy(() -> iot.searchIndex(SearchIndexRequest.builder()+ .queryString("thingName:" + prefix + "*")+ .nextToken("bogus")+ .build()))+ .isInstanceOf(InvalidRequestException.class)+ .hasMessageContaining("Invalid nextToken");+ assertThatThrownBy(() -> iot.searchIndex(SearchIndexRequest.builder()+ .queryString("thingName:" + prefix + "*")+ .queryVersion("2017-09-31")+ .build()))+ .isInstanceOf(InvalidRequestException.class)+ .hasMessageContaining("Invalid queryVersion. Expected one of: [2017-09-30]");+ } finally {+ for (String suffix : List.of("a", "b", "c")) {+ deleteIfPresent(() -> iot.deleteThing(DeleteThingRequest.builder().thingName(prefix + suffix).build()));+ }+ deleteIfPresent(() -> iot.deleteThingGroup(DeleteThingGroupRequest.builder().thingGroupName(group).build()));+ if (!TestFixtures.isRealAws()) {+ deleteIfPresent(() -> iot.deleteThingType(DeleteThingTypeRequest.builder().thingTypeName(thingType).build()));+ }+ // Indexing is account wide on AWS: put back whatever the account had before this test.+ iot.updateIndexingConfiguration(UpdateIndexingConfigurationRequest.builder()+ .thingIndexingConfiguration(prior.thingIndexingConfiguration())+ .thingGroupIndexingConfiguration(prior.thingGroupIndexingConfiguration())+ .build());+ }+ }++ /** A thing type of the account meant for tests and not deprecated, or null when it has none. */+ private String existingTestThingType() {+ return iot.listThingTypesPaginator(ListThingTypesRequest.builder().build()).thingTypes().stream()+ .filter(type -> !Boolean.TRUE.equals(type.thingTypeMetadata().deprecated()))+ .map(ThingTypeDefinition::thingTypeName)+ .filter(name -> name.contains("test"))+ .sorted()+ .findFirst()+ .orElse(null);+ }++ /**+ * This test's things a query finds, in name order. On AWS the search is polled until they are the+ * expected ones: AWS indexes a change a few seconds after it and refuses searches while a new+ * index builds. Floci answers at once, so against Floci the search runs once.+ */+ private List<ThingDocument> awaitSearch(String query, String prefix, String... expectedSuffixes)+ throws InterruptedException {+ List<String> expected = Arrays.stream(expectedSuffixes).map(suffix -> prefix + suffix).toList();+ Instant deadline = Instant.now().plusSeconds(120);+ while (true) {+ List<ThingDocument> found = new ArrayList<>();+ String notReady = null;+ try {+ String nextToken = null;+ do {+ SearchIndexResponse page = iot.searchIndex(SearchIndexRequest.builder()+ .queryString(query)+ .nextToken(nextToken)+ .build());+ page.things().stream().filter(thing -> thing.thingName().startsWith(prefix)).forEach(found::add);+ nextToken = page.nextToken();+ } while (nextToken != null);+ } catch (IndexNotReadyException e) {+ notReady = e.getMessage();+ }+ found.sort(Comparator.comparing(ThingDocument::thingName));+ List<String> names = found.stream().map(ThingDocument::thingName).toList();+ if (!TestFixtures.isRealAws() || notReady == null && names.equals(expected)+ || Instant.now().isAfter(deadline)) {+ assertThat(notReady).as("index not ready").isNull();+ assertThat(names).as(query).isEqualTo(expected);+ return found;+ }+ Thread.sleep(2_000);+ }+ }++ private static void deleteIfPresent(Runnable delete) {+ try {+ delete.run();+ } catch (ResourceNotFoundException ignored) {+ // The test failed before creating it.+ }+ }+private static void assertIndexStatus(IndexStatus status) {if (TestFixtures.isRealAws()) {// AWS builds the index in the background after a mode change; Floci has it ready at once.diff --git a/docs/services/index.md b/docs/services/index.mdindex 750246c56d..be3277207b 100644--- a/docs/services/index.md+++ b/docs/services/index.md@@ -142,7 +142,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac| [BCM Data Exports](bcm-data-exports.md) | `POST /` + `X-Amz-Target: AWSBillingAndCostManagementDataExports.*` | JSON 1.1 | 7 || [Transfer Family](transfer.md) | `POST /` + `X-Amz-Target: TransferService.*` | JSON 1.1 | 17 || [DataSync](datasync.md) | `POST /` + `X-Amz-Target: FmrsService.*` | JSON 1.1 | 48 |-| [IoT Core](iot.md) | `/things/...`, `/endpoint`, rules/policies REST paths | REST JSON | 74 |+| [IoT Core](iot.md) | `/things/...`, `/endpoint`, rules/policies REST paths | REST JSON | 75 || [IoT Data](iot.md) | `/things/{thingName}/shadow`, MQTT topics | REST JSON | 11 |**Lambda, ElastiCache, RDS, MSK, MWAA, ECS, EKS, and OpenSearch** spin up real Docker containers and support IAM authentication and SigV4 request signing, the same auth flow as production AWS. **RDS Data API** executes SQL against the local RDS containers through AWS-compatible REST JSON routes.diff --git a/docs/services/iot.md b/docs/services/iot.mdindex a156363fee..4c2dd781e7 100644--- a/docs/services/iot.md+++ b/docs/services/iot.md@@ -45,7 +45,7 @@ Current MVP 2 limitations:- `SendDirectMessage` publishes to the requested MQTT topic through the embedded broker. Unlike AWS IoT Core, it does not yet bypass subscription matching to deliver to a client that is not subscribed to that topic.- `GetConnection` and `ListSubscriptions` report live in-memory broker state only; offline persistent session subscription reporting is not modeled yet.- Jobs reserved MQTT topics remain follow-up scope; Jobs Data HTTP APIs are implemented first.-- Dynamic thing groups, job rollouts, cancellations, documents from S3, and advanced job scheduling are not yet modeled. Fleet indexing has its configuration only, see [Fleet Indexing](#fleet-indexing).+- Dynamic thing groups, job rollouts, cancellations, documents from S3, and advanced job scheduling are not yet modeled. Fleet indexing has its configuration and a bounded `SearchIndex` subset, see [Fleet Indexing](#fleet-indexing).## Domain Configurations@@ -104,7 +104,7 @@ Current limitations:## Fleet Indexing-Status: configuration only.+Status: configuration and a bounded SearchIndex subset.`UpdateIndexingConfiguration` and `GetIndexingConfiguration` (`/indexing/config`) and `DescribeIndex` (`/indices/{indexName}`) are served on the REST-JSON paths the AWS SDKs use, with the AWS shapes, error codes and validation messages:@@ -115,9 +115,50 @@ Status: configuration only.- A member of the wrong JSON type, or content after the JSON body, fails with `SerializationException`.- `DescribeIndex` reports `AWS_Things` and `AWS_ThingGroups` with the schema the modes select. The index is `ACTIVE` as soon as it is enabled: Floci has no `BUILDING` or `REBUILDING` window. A disabled index is `ResourceNotFoundException`, any other index name `InvalidRequestException`.+### SearchIndex++`SearchIndex` (`POST /indices/search`) searches the things of the caller's account and region in the `AWS_Things` index, the default index name. Floci evaluates this part of the query language:++- `field:value`. Values match case-insensitively, field names exactly: `attributes.Site:north` matches nothing when the attribute is named `site`.+- `*` (any run of characters) and `?` (one character) inside a value, and `\` to escape one character. A quoted value such as `thingName:"x"` matches literally.+- `field:*` matches the things that have the field. A bare `*` matches every thing.+- `AND`, `OR` and `NOT` in upper case, their forms `&&`, `||` and `!`, a leading `-` for negation, and parentheses.+- Precedence as measured on AWS: `NOT` binds tightest, then `AND`, then `OR`. Whitespace is an `AND` that binds looser than `OR`, so `a OR b c` means `(a OR b) AND c`.++The fields are `thingName`, `thingId`, `thingTypeName`, `thingGroupNames` (direct memberships) and `attributes.<name>`.++Errors follow AWS, with AWS's messages. The body and member types are checked first, then the member constraints, then the index, then the query, then `nextToken`:++- `InvalidRequestException` for a missing or empty `queryString`, a `queryString` longer than 1000 characters, a `maxResults` below 1 (AWS truncates it toward zero and caps it at the 32-bit integer range), an empty `indexName`, a `queryVersion` other than `2017-09-30`, an unknown index name, a `nextToken` that is empty, was not issued by `SearchIndex`, or was edited or truncated (`Invalid nextToken`), and a `nextToken` issued for a different `queryString`, `indexName` or `queryVersion` (`nextToken was from a request with different parameters`).+- `ResourceNotFoundException` for an index that is not enabled.+- `InvalidQueryException` for invalid syntax, an invalid field name (including a `shadow.name.<shadow>.<field>` whose shadow is not in the named shadow filter), more than 12 terms, more than two `*` wildcards in a value, a value starting with `?`, the `+` operator, and fuzzy, regular expression and boost queries.+- `InvalidRequestException` for a `connectivity.*`, `shadow.*` or `deviceDefender.*` field while that indexing is off. A classic `shadow.*` field needs `REGISTRY_AND_SHADOW`; a `shadow.name.*` field is also served by named shadow indexing.+- `SerializationException` for a body that is not a JSON object, malformed JSON or content after it, a `queryString`, `indexName`, `queryVersion` or `nextToken` that is not a string, and a `maxResults` that is not a number.++Within the query, invalid syntax is reported before more than 12 terms, and more than 12 terms before any other query error, including a `connectivity.*`, `shadow.*` or `deviceDefender.*` field while that indexing is off and the constructs Floci refuses below. Each value of a field group counts as a term, the field before it does not. The field of a group is checked like any other field, before the group is refused. A range is one term: two bounds, separated by spaces and optionally by `TO`, in `[` or `{` and `]` or `}`. An unclosed or otherwise malformed range, and an unescaped `[`, `]`, `{` or `}` outside a range, is invalid syntax, as on AWS.++Syntax AWS accepts but Floci does not evaluate is refused with `InvalidQueryException` and the message `Floci does not support <construct> in fleet index queries, query string: <query>`. It never answers with an empty result. This covers:++- range queries (`field:[a TO b]`, `field:{a TO b}`)+- comparisons (`>`, `<`, `>=`, `<=`)+- free text terms without a field, including lower case `and`, `or` and `not`+- field grouping (`field:(a OR b)`)+- every `connectivity.*`, `shadow.*` and `deviceDefender.*` field once its indexing is on. While that indexing is off the answer is AWS's `InvalidRequestException` above.++Searching `AWS_ThingGroups` is refused with `InvalidRequestException` and `Floci does not support searching AWS_ThingGroups` while that index is enabled. While it is disabled the answer is AWS's `ResourceNotFoundException`.++Results:++- A thing has `thingName` and `thingId`, and `thingTypeName`, `thingGroupNames` and `attributes` only when it has them. `shadow` and `deviceDefender` are never returned: Floci keeps no indexed shadow document.+- `connectivity` is never returned either. AWS adds it while `thingConnectivityIndexingMode` is `STATUS`, where a thing that never connected shows `connected` `false` and `timestamp` `0`.+- Things come in thing name order. AWS does not specify an order.+- `maxResults` and `nextToken` page the results. Without `maxResults` every match comes in one page. The token marks the last thing returned, so the next page continues after it even when things before it were deleted, as on AWS. A token continues only the request that issued it: the same `queryString`, and `indexName` and `queryVersion` sent the same way, since an omitted member differs from its explicit default. `maxResults` may change between pages.+- An index answers at once: a thing is searchable as soon as it is created, where AWS takes a few seconds.+Current limitations:-- `SearchIndex`, `ListIndices` and the statistics and aggregation APIs (`GetStatistics`, `GetCardinality`, `GetPercentiles`, `GetBucketsAggregation`) are not modeled yet.+- `ListIndices` and the statistics and aggregation APIs (`GetStatistics`, `GetCardinality`, `GetPercentiles`, `GetBucketsAggregation`) are not modeled yet.+- `customFields` types are not applied to search: attribute values are matched as strings.- `managedFields` and `customFields` sent in the thing group configuration are checked (the custom fields for their types only) but not stored: `GetIndexingConfiguration` reports the managed fields the mode selects and no custom fields.## MQTT Brokerdiff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexQuery.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexQuery.javanew file mode 100644index 0000000000..608100d8b8--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexQuery.java@@ -0,0 +1,381 @@+package io.github.hectorvent.floci.services.iot;++import com.fasterxml.jackson.databind.JsonNode;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.iot.model.IotIndexingConfiguration.ThingIndexing;++import java.util.ArrayList;+import java.util.List;+import java.util.Set;+import java.util.function.Predicate;+import java.util.regex.Matcher;+import java.util.regex.Pattern;++/**+ * The part of the AWS IoT fleet indexing query language Floci evaluates, parsed into a predicate+ * over a thing's index document. Precedence is the one measured on AWS: NOT binds tightest, then+ * AND, then OR, and whitespace is an AND looser than OR, so {@code a OR b c} is+ * {@code (a OR b) AND c}. Values match case-insensitively, field names exactly. Floci evaluates the+ * registry fields and attributes; connectivity, shadow and Device Defender fields are refused, with+ * AWS's error while their indexing is off. Syntax AWS accepts but Floci does not evaluate is refused+ * rather than matching nothing. A query is checked in the order measured on AWS: syntax, then the+ * 12-term limit, then fields and values, the first such error in the query winning.+ */+final class IotFleetIndexQuery {++ private static final Set<String> FIELDS = Set.of("thingName", "thingId", "thingTypeName", "thingGroupNames");+ private static final String ATTRIBUTES = "attributes.";+ private static final Set<String> OPERATORS = Set.of("AND", "OR", "NOT", "-", ")");+ private static final Pattern COMPARISON = Pattern.compile("[<>]");+ private static final String RANGE_BOUND = "(\"(?:\\\\.|[^\"\\\\])*\"|[^ \\]}]+)";+ private static final Pattern RANGE = Pattern.compile("[\\[{] *" + RANGE_BOUND + " +(?:TO +)?" + RANGE_BOUND+ + " *[\\]}]");+ private static final int MAX_TERMS = 12;+ private static final int MAX_WILDCARDS = 2;++ private final String queryString;+ private final ThingIndexing indexing;+ private final List<String> tokens;+ private int position;+ private int terms;+ private AwsException deferred;++ private IotFleetIndexQuery(String queryString, ThingIndexing indexing) {+ this.queryString = queryString;+ this.indexing = indexing;+ this.tokens = tokenize();+ }++ static Predicate<JsonNode> parse(String queryString, ThingIndexing indexing) {+ IotFleetIndexQuery parser = new IotFleetIndexQuery(queryString, indexing);+ Predicate<JsonNode> query = parser.clauses();+ if (parser.position < parser.tokens.size()) {+ throw parser.syntaxError();+ }+ if (parser.terms > MAX_TERMS) {+ throw parser.parseError("number of terms cannot exceed " + MAX_TERMS + ", terms found: " + parser.terms);+ }+ if (parser.deferred != null) {+ throw parser.deferred;+ }+ return query;+ }++ /**+ * Words, quoted runs kept inside their word, parentheses, and a leading - or ! as negation. A+ * range opened right after a field's colon, or at the start of a word, runs to its closing+ * bracket as one word.+ */+ private List<String> tokenize() {+ List<String> found = new ArrayList<>();+ int i = 0;+ while (i < queryString.length()) {+ char c = queryString.charAt(i);+ if (Character.isWhitespace(c)) {+ i++;+ } else if (c == '(' || c == ')') {+ found.add(String.valueOf(c));+ i++;+ } else if (c == '-' || c == '!') {+ found.add("-");+ i++;+ } else {+ int start = i;+ boolean quoted = false;+ boolean range = false;+ while (i < queryString.length()) {+ char next = queryString.charAt(i);+ if (next == '\\') {+ i++;+ } else if (next == '"') {+ quoted = !quoted;+ } else if (!quoted && range && (next == ']' || next == '}')) {+ i++;+ break;+ } else if (!quoted && !range && (next == '[' || next == '{')+ && (i == start || queryString.charAt(i - 1) == ':')) {+ range = true;+ } else if (!quoted && !range && (Character.isWhitespace(next) || next == '(' || next == ')')) {+ break;+ }+ i++;+ }+ if (quoted) {+ throw syntaxError();+ }+ String word = queryString.substring(start, Math.min(i, queryString.length()));+ found.add(switch (word) {+ case "&&" -> "AND";+ case "||" -> "OR";+ default -> word;+ });+ }+ }+ return found;+ }++ /** Clauses joined by whitespace, which AWS treats as an AND binding looser than OR. */+ private Predicate<JsonNode> clauses() {+ Predicate<JsonNode> query = or();+ while (position < tokens.size() && !")".equals(tokens.get(position))) {+ query = query.and(or());+ }+ return query;+ }++ private Predicate<JsonNode> or() {+ Predicate<JsonNode> query = and();+ while (accept("OR")) {+ query = query.or(and());+ }+ return query;+ }++ private Predicate<JsonNode> and() {+ Predicate<JsonNode> query = unary();+ while (accept("AND")) {+ query = query.and(unary());+ }+ return query;+ }++ private Predicate<JsonNode> unary() {+ if (accept("NOT") || accept("-")) {+ return primary().negate();+ }+ return primary();+ }++ private Predicate<JsonNode> primary() {+ if (position >= tokens.size()) {+ throw syntaxError();+ }+ String token = tokens.get(position++);+ if ("(".equals(token)) {+ Predicate<JsonNode> group = clauses();+ if (!accept(")")) {+ throw syntaxError();+ }+ return group;+ }+ if (OPERATORS.contains(token)) {+ throw syntaxError();+ }+ return term(token);+ }++ private Predicate<JsonNode> term(String token) {+ int colon = token.indexOf(':');+ String value = colon < 0 ? "" : token.substring(colon + 1);+ if (colon >= 0 && value.isEmpty()) {+ if (hasBracket(token) || !accept("(")) {+ throw syntaxError();+ }+ // AWS counts each value of a field group as a term, but not the field, and checks the field first.+ Predicate<JsonNode> refused;+ try {+ checkField(token.substring(0, colon));+ refused = defer(unsupported("field grouping"));+ } catch (AwsException e) {+ refused = defer(e);+ }+ clauses();+ if (!accept(")")) {+ throw syntaxError();+ }+ return refused;+ }+ terms++;+ boolean quoted = value.startsWith("\"");+ String rangeText = colon < 0 ? token : value;+ boolean range = rangeText.startsWith("[") || rangeText.startsWith("{");+ if ((quoted && (value.length() < 2 || !value.endsWith("\"")))+ || (!quoted && (value.startsWith("<") || value.startsWith(">")))+ || hasBracket(range ? token.substring(0, token.length() - rangeText.length()) : token)+ || (range && !isRange(rangeText))) {+ throw syntaxError();+ }+ try {+ return match(token, colon, value);+ } catch (AwsException e) {+ return defer(e);+ }+ }++ private Predicate<JsonNode> match(String token, int colon, String value) {+ if ("*".equals(token)) {+ return document -> true;+ }+ if (token.startsWith("+")) {+ throw parseError("unsupported operator - \"+\"");+ }+ Matcher comparison = COMPARISON.matcher(token);+ if (comparison.find() && (colon < 0 || comparison.start() < colon)) {+ checkField(token.substring(0, comparison.start()));+ throw unsupported("comparisons");+ }+ if (colon < 0) {+ throw unsupported("free text terms");+ }+ String field = token.substring(0, colon);+ checkField(field);+ Pattern pattern = valuePattern(field, value);+ return document -> {+ int dot = field.indexOf('.');+ JsonNode node = dot < 0 ? document.path(field)+ : document.path(field.substring(0, dot)).path(field.substring(dot + 1));+ Iterable<JsonNode> values = node.isArray() ? node : node.isMissingNode() ? List.of() : List.of(node);+ for (JsonNode candidate : values) {+ if (pattern.matcher(candidate.asText()).matches()) {+ return true;+ }+ }+ return false;+ };+ }++ /**+ * A registry field or attribute passes. One under connectivity, shadow or Device Defender first+ * needs that indexing on, with AWS's error: a classic shadow field needs REGISTRY_AND_SHADOW, a+ * named shadow field needs named shadow indexing and a shadow in its filter (otherwise an invalid+ * field name, as on AWS). Floci then refuses it, since it indexes none of them. Anything else is+ * the invalid field name AWS reports.+ */+ private void checkField(String field) {+ if (field.startsWith(ATTRIBUTES) && field.length() > ATTRIBUTES.length()) {+ return;+ }+ String attribute;+ boolean enabled;+ if (field.startsWith("connectivity.")) {+ attribute = "Connectivity";+ enabled = "STATUS".equals(indexing.thingConnectivityIndexingMode());+ } else if (field.startsWith("shadow.")) {+ attribute = "Shadow";+ boolean named = "shadow.name".equals(field) || field.startsWith("shadow.name.");+ enabled = "REGISTRY_AND_SHADOW".equals(indexing.thingIndexingMode())+ || named && "ON".equals(indexing.namedShadowIndexingMode());+ if (enabled && named) {+ // shadow.name.<shadow>.<field> exists only for a shadow in the named shadow filter.+ String[] parts = field.split("\\.", 4);+ if (parts.length < 4 || !"ON".equals(indexing.namedShadowIndexingMode())+ || !indexing.namedShadowNames().contains(parts[2])) {+ throw parseError("invalid field name, field name: " + field);+ }+ }+ } else if (field.startsWith("deviceDefender.")) {+ attribute = "Devicedefender";+ enabled = "VIOLATIONS".equals(indexing.deviceDefenderIndexingMode());+ } else if (FIELDS.contains(field)) {+ return;+ } else {+ throw parseError("invalid field name, field name: " + field);+ }+ if (!enabled) {+ throw new AwsException("InvalidRequestException", "Query includes one or more constraints for "+ + attribute + " attribute, but " + attribute + " indexing is not enabled for AWS_Things index", 400);+ }+ if (!FIELDS.contains(field)) {+ throw unsupported("the field " + field);+ }+ }++ /**+ * A quoted value matches literally; otherwise * and ? are wildcards and a backslash escapes.+ * As on AWS an unquoted value takes at most two unescaped * and cannot start with an unescaped ?.+ */+ private Pattern valuePattern(String field, String value) {+ boolean quoted = value.startsWith("\"");+ if (!quoted) {+ switch (value.charAt(0)) {+ case '[', '{' -> throw unsupported("range queries");+ case '/' -> throw parseError("unsupported query type - regular expression");+ case '?' -> throw invalidValue("wildcard character (?) is not allowed at the start of the value",+ field, value);+ default -> {+ }+ }+ }+ String text = quoted ? value.substring(1, value.length() - 1) : value;+ StringBuilder regex = new StringBuilder();+ int wildcards = 0;+ for (int i = 0; i < text.length(); i++) {+ char c = text.charAt(i);+ if (c == '\\' && i + 1 < text.length()) {+ regex.append(Pattern.quote(String.valueOf(text.charAt(++i))));+ } else if (!quoted && c == '*') {+ wildcards++;+ regex.append(".*");+ } else if (!quoted && c == '?') {+ regex.append('.');+ } else if (!quoted && c == '~') {+ throw parseError("unsupported query type - fuzzy");+ } else if (!quoted && c == '^') {+ throw parseError("unsupported query type - boost");+ } else {+ regex.append(Pattern.quote(String.valueOf(c)));+ }+ }+ if (wildcards > MAX_WILDCARDS) {+ throw invalidValue("too many wildcards (*). The maximum allowed per term is " + MAX_WILDCARDS,+ field, value);+ }+ return Pattern.compile(regex.toString(), Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.DOTALL);+ }++ /** A [, ], { or } that is neither escaped nor quoted, which AWS's parser rejects outside a range. */+ private static boolean hasBracket(String text) {+ boolean quoted = false;+ for (int i = 0; i < text.length(); i++) {+ char c = text.charAt(i);+ if (c == '\\') {+ i++;+ } else if (c == '"') {+ quoted = !quoted;+ } else if (!quoted && "[]{}".indexOf(c) >= 0) {+ return true;+ }+ }+ return false;+ }++ /** Two bounds, optionally joined by TO, in [ or { and ] or }, where TO itself is no bound. */+ private static boolean isRange(String text) {+ Matcher range = RANGE.matcher(text);+ return range.matches() && !"TO".equals(range.group(1)) && !"TO".equals(range.group(2));+ }++ /** Records the first error AWS reports only once the query parses and its terms are counted. */+ private Predicate<JsonNode> defer(AwsException error) {+ if (deferred == null) {+ deferred = error;+ }+ return document -> false;+ }++ private boolean accept(String token) {+ if (position < tokens.size() && token.equals(tokens.get(position))) {+ position++;+ return true;+ }+ return false;+ }++ private AwsException syntaxError() {+ return parseError("invalid syntax");+ }++ private AwsException parseError(String reason) {+ return new AwsException("InvalidQueryException",+ "Unable to parse query, " + reason + ", query string: " + queryString, 400);+ }++ private AwsException invalidValue(String reason, String field, String value) {+ return parseError("invalid value - " + reason + ", field name: " + field + ", value: " + value);+ }++ private AwsException unsupported(String construct) {+ return new AwsException("InvalidQueryException",+ "Floci does not support " + construct + " in fleet index queries, query string: " + queryString, 400);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingController.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingController.javaindex 6a1aedf14e..8b84b49a16 100644--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingController.java+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingController.java@@ -22,7 +22,7 @@ import java.util.List;/*** REST-JSON routes for AWS IoT fleet indexing: UpdateIndexingConfiguration,- * GetIndexingConfiguration and DescribeIndex, on the paths and shapes the AWS SDKs use.+ * GetIndexingConfiguration, DescribeIndex and SearchIndex, on the paths and shapes the AWS SDKs use.*/@Path("/")@Produces(MediaType.APPLICATION_JSON)@@ -93,4 +93,17 @@ public class IotFleetIndexingController {response.put("schema", schema);return Response.ok(response).build();}++ @POST+ @Path("/indices/search")+ public Response searchIndex(@Context HttpHeaders headers, String body) {+ IotService.Page<ObjectNode> page = fleetIndexingService.searchIndex(IotRequestBody.read(objectMapper, body),+ regionResolver.resolveRegion(headers));+ ObjectNode response = objectMapper.createObjectNode();+ response.putArray("things").addAll(page.items());+ if (page.nextToken() != null) {+ response.put("nextToken", page.nextToken());+ }+ return Response.ok(response).build();+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingService.java b/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingService.javaindex ff4850ab67..659b1e6284 100644--- a/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingService.java+++ b/src/main/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingService.java@@ -2,8 +2,10 @@ package io.github.hectorvent.floci.services.iot;import com.fasterxml.jackson.core.type.TypeReference;import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.node.JsonNodeFactory;import com.fasterxml.jackson.databind.node.JsonNodeType;import com.fasterxml.jackson.databind.node.MissingNode;+import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;@@ -11,21 +13,30 @@ import io.github.hectorvent.floci.services.iot.model.IotIndexingConfiguration;import io.github.hectorvent.floci.services.iot.model.IotIndexingConfiguration.Field;import io.github.hectorvent.floci.services.iot.model.IotIndexingConfiguration.GeoLocation;import io.github.hectorvent.floci.services.iot.model.IotIndexingConfiguration.ThingIndexing;+import io.github.hectorvent.floci.services.iot.model.IotThingGroup;+import io.github.hectorvent.floci.services.iot.model.Thing;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;+import java.nio.charset.StandardCharsets;import java.util.ArrayList;+import java.util.Base64;import java.util.Collection;+import java.util.HashMap;import java.util.List;import java.util.Locale;import java.util.Map;import java.util.Set;+import java.util.UUID;+import java.util.function.Predicate;+import java.util.regex.Matcher;+import java.util.regex.Pattern;import java.util.stream.Collectors;import java.util.stream.Stream;/**- * AWS IoT fleet indexing configuration: UpdateIndexingConfiguration, GetIndexingConfiguration and- * DescribeIndex. One configuration per account and region; a region never updated is OFF.+ * AWS IoT fleet indexing: UpdateIndexingConfiguration, GetIndexingConfiguration, DescribeIndex and+ * a bounded SearchIndex. One configuration per account and region; a region never updated is OFF.*/@ApplicationScopedpublic class IotFleetIndexingService {@@ -33,6 +44,9 @@ public class IotFleetIndexingService {static final String THINGS_INDEX = "AWS_Things";static final String THING_GROUPS_INDEX = "AWS_ThingGroups";+ private static final int MAX_QUERY_STRING_LENGTH = 1000;+ private static final Pattern PAGE_TOKEN = Pattern.compile("([0-9a-f-]{36})([0-9a-f-]{36})(.+)", Pattern.DOTALL);+// Enum value sets in the order AWS prints them in its validation messages.private static final List<String> THING_INDEXING_MODES = List.of("REGISTRY_AND_SHADOW", "OFF", "REGISTRY");private static final List<String> CONNECTIVITY_MODES = List.of("OFF", "STATUS");@@ -81,17 +95,19 @@ public class IotFleetIndexingService {new Field("thingGroupId", "String"));private final StorageBackend<String, IotIndexingConfiguration> store;+ private final IotService iotService;/** Guards the read-modify-write of an update that changes only one of the two configurations. */private final Object lock = new Object();@Inject- public IotFleetIndexingService(StorageFactory storageFactory) {+ public IotFleetIndexingService(StorageFactory storageFactory, IotService iotService) {this(storageFactory.create("iot", "iot-indexing-configuration.json",- new TypeReference<Map<String, IotIndexingConfiguration>>() {}));+ new TypeReference<Map<String, IotIndexingConfiguration>>() {}), iotService);}- IotFleetIndexingService(StorageBackend<String, IotIndexingConfiguration> store) {+ IotFleetIndexingService(StorageBackend<String, IotIndexingConfiguration> store, IotService iotService) {this.store = store;+ this.iotService = iotService;}public void updateIndexingConfiguration(JsonNode request, String region) {@@ -114,10 +130,7 @@ public class IotFleetIndexingService {List<Field> groupManagedFields = fields(groupNode.path("managedFields"), groupPrefix + ".managedFields",errors);fields(groupNode.path("customFields"), groupPrefix + ".customFields", errors);- if (!errors.isEmpty()) {- throw invalid(errors.size() + (errors.size() == 1 ? " validation error" : " validation errors")- + " detected: " + String.join("; ", errors));- }+ rejectViolations(errors);if (thing != null) {thing = validateThing(thing, managedFields);}@@ -137,16 +150,168 @@ public class IotFleetIndexingService {/** The schema of an enabled index. */public String describeIndex(String indexName, String region) {+ String schema = schema(indexName, getIndexingConfiguration(region));+ if (schema == null) {+ throw new AwsException("ResourceNotFoundException", "Index " + indexName + " does not exist", 404);+ }+ return schema;+ }++ /**+ * SearchIndex over the things of the caller's account and region, in thing name order. A+ * request is checked in the order measured on AWS: the body and member types, member+ * constraints, an empty index name, the query version, an unknown or disabled index, the query,+ * then the page token. The token names the request it continues (its query string, index name+ * and version as sent) and the last thing returned, so a page continues after that thing+ * whatever was deleted before it. It also carries a checksum, so an edited token is invalid.+ */+ public IotService.Page<ObjectNode> searchIndex(JsonNode request, String region) {+ if (!request.isObject()) {+ throw serializationError("request body", JsonNodeType.OBJECT);+ }+ String queryString = text(request.path("queryString"), "queryString");+ JsonNode maxResults = request.path("maxResults");+ if (present(maxResults) && !maxResults.isNumber()) {+ throw serializationError("maxResults", JsonNodeType.NUMBER);+ }+ // AWS truncates maxResults toward zero and saturates it at the int range, as this cast does.+ int pageSize = present(maxResults) ? (int) maxResults.asDouble() : Integer.MAX_VALUE;+ String requestedIndexName = text(request.path("indexName"), "indexName");+ String queryVersion = text(request.path("queryVersion"), "queryVersion");+ String nextToken = text(request.path("nextToken"), "nextToken");+ List<String> errors = new ArrayList<>();+ if (queryString == null) {+ errors.add("Value null at 'queryString' failed to satisfy constraint: Member must not be null");+ } else if (queryString.isEmpty()) {+ errors.add("Value '' at 'queryString' failed to satisfy constraint: "+ + "Member must have length greater than or equal to 1");+ } else if (queryString.codePointCount(0, queryString.length()) > MAX_QUERY_STRING_LENGTH) {+ errors.add("Value '" + queryString + "' at 'queryString' failed to satisfy constraint: "+ + "Member must have length less than or equal to " + MAX_QUERY_STRING_LENGTH);+ }+ if (pageSize < 1) {+ errors.add("Value '" + pageSize + "' at 'maxResults' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 1");+ }+ rejectViolations(errors);+ String indexName = requestedIndexName == null ? THINGS_INDEX : requestedIndexName;+ if (indexName.isEmpty()) {+ throw invalid("indexName cannot be empty.");+ }+ if (queryVersion != null && !"2017-09-30".equals(queryVersion)) {+ throw invalid("Invalid queryVersion. Expected one of: [2017-09-30]");+ }IotIndexingConfiguration configuration = getIndexingConfiguration(region);- String schema = switch (indexName) {+ if (schema(indexName, configuration) == null) {+ throw new AwsException("ResourceNotFoundException", "Index " + indexName+ + " does not exist. Please enable index by calling UpdateIndexingConfiguration", 404);+ }+ if (THING_GROUPS_INDEX.equals(indexName)) {+ // ponytail: thing group documents are not modeled; refusing beats an empty result.+ throw invalid("Floci does not support searching AWS_ThingGroups");+ }+ ThingIndexing indexing = configuration.thing();+ Predicate<JsonNode> query = IotFleetIndexQuery.parse(queryString, indexing);+ String parameters = pageParameters(requestedIndexName, queryVersion, queryString);+ String after = pageAfter(nextToken, parameters);+ Map<String, List<String>> thingGroupNames = thingGroupNames(region);+ List<ObjectNode> matches = new ArrayList<>();+ for (Thing thing : iotService.listThings(region)) {+ String thingName = thing.getThingName();+ if (after != null && thingName.compareTo(after) <= 0) {+ continue;+ }+ ObjectNode document = document(thing, thingGroupNames.getOrDefault(thingName, List.of()));+ if (query.test(document)) {+ matches.add(document);+ if (matches.size() > pageSize) {+ break;+ }+ }+ }+ if (matches.size() <= pageSize) {+ return new IotService.Page<>(matches, null);+ }+ matches.removeLast();+ return new IotService.Page<>(matches, pageToken(parameters, matches.getLast().path("thingName").asText()));+ }++ /**+ * A digest of the members a page token is bound to, each as sent: AWS treats an omitted index+ * name or version as different from its explicit default.+ */+ private static String pageParameters(String indexName, String queryVersion, String queryString) {+ // The index name and version were validated against closed sets, so only the query can hold a newline.+ return UUID.nameUUIDFromBytes((indexName + "\n" + queryVersion + "\n" + queryString)+ .getBytes(StandardCharsets.UTF_8)).toString();+ }++ /** The thing name a page token continues after, null for the first page. */+ private static String pageAfter(String nextToken, String parameters) {+ if (nextToken == null) {+ return null;+ }+ Matcher token;+ try {+ token = PAGE_TOKEN.matcher(new String(Base64.getUrlDecoder().decode(nextToken), StandardCharsets.UTF_8));+ } catch (IllegalArgumentException e) {+ throw invalid("Invalid nextToken");+ }+ if (!token.matches() || !token.group(2).equals(pageCheck(token.group(1), token.group(3)))) {+ throw invalid("Invalid nextToken");+ }+ if (!token.group(1).equals(parameters)) {+ throw invalid("nextToken was from a request with different parameters");+ }+ return token.group(3);+ }++ private static String pageToken(String parameters, String thingName) {+ return Base64.getUrlEncoder().withoutPadding().encodeToString(+ (parameters + pageCheck(parameters, thingName) + thingName).getBytes(StandardCharsets.UTF_8));+ }++ /** A checksum of a token's parameters and cursor, so an edited or truncated token is invalid, as on AWS. */+ private static String pageCheck(String parameters, String thingName) {+ return UUID.nameUUIDFromBytes((parameters + thingName).getBytes(StandardCharsets.UTF_8)).toString();+ }++ /** The thing groups each thing of the region is a direct member of, in group name order. */+ private Map<String, List<String>> thingGroupNames(String region) {+ Map<String, List<String>> groups = new HashMap<>();+ for (IotThingGroup group : iotService.listThingGroups(region, null, null).items()) {+ for (String thingName : iotService.listThingsInThingGroup(group.getThingGroupName(), region)) {+ groups.computeIfAbsent(thingName, ignored -> new ArrayList<>()).add(group.getThingGroupName());+ }+ }+ return groups;+ }++ /** A thing's index document; like AWS it leaves out a member the thing does not have. */+ private static ObjectNode document(Thing thing, List<String> thingGroupNames) {+ ObjectNode document = JsonNodeFactory.instance.objectNode();+ document.put("thingName", thing.getThingName());+ document.put("thingId", thing.getThingId());+ if (thing.getThingTypeName() != null) {+ document.put("thingTypeName", thing.getThingTypeName());+ }+ if (!thingGroupNames.isEmpty()) {+ thingGroupNames.forEach(document.putArray("thingGroupNames")::add);+ }+ if (thing.getAttributes() != null && !thing.getAttributes().isEmpty()) {+ ObjectNode attributes = document.putObject("attributes");+ thing.getAttributes().forEach(attributes::put);+ }+ return document;+ }++ /** The schema of the named index, null while it is disabled. */+ private static String schema(String indexName, IotIndexingConfiguration configuration) {+ return switch (indexName) {case THINGS_INDEX -> thingsSchema(configuration.thing());case THING_GROUPS_INDEX -> "OFF".equals(configuration.thingGroupIndexingMode()) ? null : "REGISTRY";default -> throw invalid("Unrecognized indexName " + indexName);};- if (schema == null) {- throw new AwsException("ResourceNotFoundException", "Index " + indexName + " does not exist", 404);- }- return schema;}/** The managed fields AWS derives from the thing indexing modes; none while indexing is OFF. */@@ -289,6 +454,13 @@ public class IotFleetIndexingService {return value;}+ private static void rejectViolations(List<String> errors) {+ if (!errors.isEmpty()) {+ throw invalid(errors.size() + (errors.size() == 1 ? " validation error" : " validation errors")+ + " detected: " + String.join("; ", errors));+ }+ }+private static String enumError(String value, String path, List<String> allowed) {return "Value '" + value + "' at '" + path+ "' failed to satisfy constraint: Member must satisfy enum value set: " + allowed;diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingIntegrationTest.javaindex f75a03282c..790e5c242d 100644--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingIntegrationTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.iot;import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.ValidatableResponse;import org.junit.jupiter.api.Test;import static io.restassured.RestAssured.given;@@ -9,17 +10,20 @@ import static org.hamcrest.Matchers.anEmptyMap;import static org.hamcrest.Matchers.contains;import static org.hamcrest.Matchers.containsInAnyOrder;import static org.hamcrest.Matchers.empty;+import static org.hamcrest.Matchers.emptyOrNullString;import static org.hamcrest.Matchers.equalTo;import static org.hamcrest.Matchers.hasItems;import static org.hamcrest.Matchers.hasKey;import static org.hamcrest.Matchers.hasSize;import static org.hamcrest.Matchers.not;+import static org.hamcrest.Matchers.notNullValue;import static org.hamcrest.Matchers.startsWith;/*** Drives fleet indexing the way the AWS SDKs do: UpdateIndexingConfiguration and- * GetIndexingConfiguration on {@code /indexing/config}, DescribeIndex on {@code /indices/{indexName}}.- * Every test signs as its own account, so the configurations they store never meet.+ * GetIndexingConfiguration on {@code /indexing/config}, DescribeIndex on {@code /indices/{indexName}}+ * and SearchIndex on {@code /indices/search}. Every test signs as its own account, so the+ * configurations and things they store never meet.*/@QuarkusTestclass IotFleetIndexingIntegrationTest {@@ -283,4 +287,208 @@ class IotFleetIndexingIntegrationTest {.statusCode(200).body("thingIndexingConfiguration.thingIndexingMode", equalTo("OFF"));}++ private static void post(String auth, String path, String body) {+ given()+ .header("Authorization", auth)+ .contentType("application/json")+ .body(body)+ .when()+ .post(path)+ .then()+ .statusCode(200);+ }++ private static ValidatableResponse search(String auth, String body) {+ return given()+ .header("Authorization", auth)+ .contentType("application/json")+ .body(body)+ .when()+ .post("/indices/search")+ .then();+ }++ @Test+ void searchIndexReturnsTheSdkWireShape() {+ String auth = auth("111100000013", "us-east-1");+ update(auth, """+ {"thingIndexingConfiguration": {"thingIndexingMode": "REGISTRY", "thingConnectivityIndexingMode": "STATUS"}}+ """);+ post(auth, "/thing-types/search-sensor", "{}");+ post(auth, "/thing-groups/search-north", "{}");+ post(auth, "/things/search-a", """+ {"thingTypeName": "search-sensor", "attributePayload": {"attributes": {"provider": "Acme", "site": "north"}}}+ """);+ post(auth, "/things/search-b", """+ {"attributePayload": {"attributes": {"provider": "acme"}}}+ """);+ post(auth, "/things/search-c", "{}");+ given()+ .header("Authorization", auth)+ .contentType("application/json")+ .body("""+ {"thingGroupName": "search-north", "thingName": "search-a"}+ """)+ .when()+ .put("/thing-groups/addThingToThingGroup")+ .then()+ .statusCode(200);++ search(auth, """+ {"queryString": "attributes.provider:acme AND thingTypeName:search-sensor"}+ """)+ .statusCode(200)+ .body("$", aMapWithSize(1))+ .body("things", hasSize(1))+ .body("things[0]", aMapWithSize(5))+ .body("things[0].thingName", equalTo("search-a"))+ .body("things[0].thingId", not(emptyOrNullString()))+ .body("things[0].thingTypeName", equalTo("search-sensor"))+ .body("things[0].thingGroupNames", contains("search-north"))+ .body("things[0].attributes.provider", equalTo("Acme"))+ .body("things[0].attributes.site", equalTo("north"))+ .body("things[0]", not(hasKey("connectivity")));+ search(auth, """+ {"queryString": "thingName:search-c", "queryVersion": "2017-09-30", "indexName": "AWS_Things"}+ """)+ .statusCode(200)+ .body("things[0]", aMapWithSize(2))+ .body("things[0].thingName", equalTo("search-c"))+ .body("things[0]", not(hasKey("connectivity")));+ search(auth, """+ {"queryString": "connectivity.connected:true"}+ """)+ .statusCode(400)+ .body("__type", equalTo("InvalidQueryException"))+ .body("message", equalTo("Floci does not support the field connectivity.connected in fleet index "+ + "queries, query string: connectivity.connected:true"));++ String nextToken = search(auth, """+ {"queryString": "thingName:search-*", "maxResults": 2}+ """)+ .statusCode(200)+ .body("things.thingName", contains("search-a", "search-b"))+ .body("nextToken", notNullValue())+ .extract().path("nextToken");+ search(auth, "{\"queryString\": \"thingName:search-*\", \"maxResults\": 2, \"nextToken\": \"" + nextToken + "\"}")+ .statusCode(200)+ .body("things.thingName", contains("search-c"))+ .body("$", not(hasKey("nextToken")));+ }++ @Test+ void searchIndexRejectsBadRequestsWithAwsErrors() {+ String auth = auth("111100000008", "us-east-1");+ update(auth, """+ {"thingIndexingConfiguration": {"thingIndexingMode": "REGISTRY"}}+ """);++ search(auth, "{}")+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"))+ .body("message", equalTo("1 validation error detected: Value null at 'queryString' failed to satisfy "+ + "constraint: Member must not be null"));+ search(auth, """+ {"queryString": "*", "queryVersion": "2017-09-31"}+ """)+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"))+ .body("message", equalTo("Invalid queryVersion. Expected one of: [2017-09-30]"));+ search(auth, """+ {"queryString": "*", "nextToken": "bogus"}+ """)+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"))+ .body("message", equalTo("Invalid nextToken"));+ search(auth, """+ {"queryString": "ThingName:x"}+ """)+ .statusCode(400)+ .body("__type", equalTo("InvalidQueryException"))+ .body("message", equalTo("Unable to parse query, invalid field name, field name: ThingName, "+ + "query string: ThingName:x"));+ search(auth, """+ {"queryString": "thingName>a"}+ """)+ .statusCode(400)+ .body("__type", equalTo("InvalidQueryException"))+ .body("message", equalTo("Floci does not support comparisons in fleet index queries, "+ + "query string: thingName>a"));+ search(auth, """+ {"queryString": "connectivity.connected:true"}+ """)+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"))+ .body("message", equalTo("Query includes one or more constraints for Connectivity attribute, "+ + "but Connectivity indexing is not enabled for AWS_Things index"));+ search(auth, """+ {"queryString": "*", "maxResults": 9223372036854775808}+ """)+ .statusCode(200);+ search(auth, """+ {"queryString": "*", "maxResults": -9223372036854775809}+ """)+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"))+ .body("message", equalTo("1 validation error detected: Value '-2147483648' at 'maxResults' failed to "+ + "satisfy constraint: Member must have value greater than or equal to 1"));+ }++ @Test+ void searchIndexOfADisabledIndexIsNotFound() {+ String auth = auth("111100000009", "us-east-1");++ search(auth, """+ {"queryString": "*"}+ """)+ .statusCode(404)+ .body("__type", equalTo("ResourceNotFoundException"))+ .body("message", equalTo("Index AWS_Things does not exist. "+ + "Please enable index by calling UpdateIndexingConfiguration"));+ search(auth, """+ {"queryString": "*", "indexName": "AWS_ThingGroups"}+ """)+ .statusCode(404)+ .body("__type", equalTo("ResourceNotFoundException"))+ .body("message", equalTo("Index AWS_ThingGroups does not exist. "+ + "Please enable index by calling UpdateIndexingConfiguration"));++ update(auth, """+ {"thingGroupIndexingConfiguration": {"thingGroupIndexingMode": "ON"}}+ """);+ search(auth, """+ {"queryString": "*", "indexName": "AWS_ThingGroups"}+ """)+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"))+ .body("message", equalTo("Floci does not support searching AWS_ThingGroups"));+ }++ @Test+ void searchIndexIsIsolatedPerAccountAndRegion() {+ String home = auth("111100000010", "us-east-1");+ String otherAccount = auth("111100000011", "us-east-1");+ String otherRegion = auth("111100000010", "eu-west-1");+ String indexing = """+ {"thingIndexingConfiguration": {"thingIndexingMode": "REGISTRY"}}+ """;+ update(home, indexing);+ update(otherAccount, indexing);+ update(otherRegion, indexing);+ post(home, "/things/isolated-thing", "{}");+ String query = """+ {"queryString": "thingName:isolated-thing"}+ """;++ search(home, query)+ .statusCode(200)+ .body("things.thingName", contains("isolated-thing"));+ search(otherAccount, query)+ .statusCode(200)+ .body("things", empty());+ search(otherRegion, query)+ .statusCode(200)+ .body("things", empty());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingServiceTest.javaindex 34699f60f3..2c0dcb9730 100644--- a/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/iot/IotFleetIndexingServiceTest.java@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.iot;import com.fasterxml.jackson.core.JsonProcessingException;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.iot.model.IotIndexingConfiguration;@@ -14,13 +15,22 @@ import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.CsvSource;import org.junit.jupiter.params.provider.ValueSource;+import java.math.BigDecimal;+import java.math.BigInteger;+import java.nio.charset.StandardCharsets;+import java.util.Base64;+import java.util.Collections;import java.util.HashSet;import java.util.List;import java.util.Map;import java.util.Set;+import java.util.stream.IntStream;import java.util.stream.Stream;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;@@ -62,8 +72,13 @@ class IotFleetIndexingServiceTest {new Field("thingGroupName", "String"),new Field("thingGroupId", "String"));+ private static final String CONNECTIVITY_NOT_ENABLED = "Query includes one or more constraints for Connectivity "+ + "attribute, but Connectivity indexing is not enabled for AWS_Things index";+private final ObjectMapper mapper = new ObjectMapper();- private final IotFleetIndexingService service = new IotFleetIndexingService(new InMemoryStorage<>());+ private final IotServiceTestSupport iot = new IotServiceTestSupport(REGION, null);+ private final IotFleetIndexingService service =+ new IotFleetIndexingService(new InMemoryStorage<>(), iot.service);private JsonNode json(String text) {try {@@ -675,4 +690,640 @@ class IotFleetIndexingServiceTest {""");assertDescribeFails("AWS_Thing", "InvalidRequestException", 400, "Unrecognized indexName AWS_Thing");}++ /**+ * Four things in the default region: alpha-1 and beta-1 of type sensor, alpha-1 and alpha-2 in+ * thing groups, attribute values in mixed case, and gamma with nothing but its name.+ */+ private void fleet(String thingIndexing) {+ update("{\"thingIndexingConfiguration\": " + thingIndexing + "}");+ IotService registry = iot.service;+ registry.createThingType("sensor", mapper.createObjectNode(), REGION);+ registry.createThingGroup("north-group", mapper.createObjectNode(), REGION);+ registry.createThingGroup("south-group", mapper.createObjectNode(), REGION);+ registry.createThing("alpha-1", Map.of("site", "North", "provider", "acme"), "sensor", REGION);+ registry.createThing("alpha-2", Map.of("site", "south", "provider", "ACME"), null, REGION);+ registry.createThing("beta-1", Map.of("provider", "other"), "sensor", REGION);+ registry.createThing("gamma", Map.of(), null, REGION);+ registry.addThingToThingGroup("north-group", "alpha-1", REGION);+ registry.addThingToThingGroup("south-group", "alpha-2", REGION);+ registry.addThingToThingGroup("north-group", "alpha-2", REGION);+ }++ private void fleet() {+ fleet("{\"thingIndexingMode\": \"REGISTRY\"}");+ }++ private ObjectNode query(String queryString) {+ return mapper.createObjectNode().put("queryString", queryString);+ }++ private IotService.Page<ObjectNode> search(ObjectNode request) {+ return service.searchIndex(request, REGION);+ }++ private List<String> names(String queryString) {+ return search(query(queryString)).items().stream().map(thing -> thing.path("thingName").asText()).toList();+ }++ private ObjectNode document(String thingName) {+ List<ObjectNode> things = search(query("thingName:" + thingName)).items();+ assertEquals(1, things.size());+ return things.get(0);+ }++ private void assertSearchFails(ObjectNode request, String code, int status, String message) {+ AwsException failure = assertThrows(AwsException.class, () -> search(request));+ assertEquals(code, failure.getErrorCode());+ assertEquals(status, failure.getHttpStatus());+ assertEquals(message, failure.getMessage());+ }++ private void assertQueryFails(String queryString, String message) {+ assertSearchFails(query(queryString), "InvalidQueryException", 400, message + ", query string: " + queryString);+ }++ private void assertUnsupported(String queryString, String construct) {+ assertQueryFails(queryString, "Floci does not support " + construct + " in fleet index queries");+ }++ @Test+ void searchMatchesValuesCaseInsensitivelyInThingNameOrder() {+ fleet();++ assertEquals(List.of("alpha-1"), names("thingName:ALPHA-1"));+ assertEquals(List.of("alpha-1"), names("attributes.site:north"));+ assertEquals(List.of("alpha-1", "alpha-2"), names("attributes.provider:Acme"));+ assertEquals(List.of("alpha-1", "beta-1"), names("thingTypeName:SENSOR"));+ assertEquals(List.of("alpha-1", "alpha-2"), names("thingGroupNames:north-group"));+ assertEquals(List.of("alpha-2"), names("thingGroupNames:south-group"));+ assertEquals(List.of("gamma"), names("thingId:" + iot.service.describeThing("gamma", REGION).getThingId()));+ assertEquals(List.of("alpha-1"), names("thingName:\"Alpha-1\""));+ assertEquals(List.of(), names("thingName:alpha"));+ assertEquals(List.of(), names("attributes.missing:x"));+ }++ @Test+ void fieldNamesAreCaseSensitive() {+ fleet();++ assertEquals(List.of(), names("attributes.Site:north"));+ assertQueryFails("ThingName:alpha-1", "Unable to parse query, invalid field name, field name: ThingName");+ assertQueryFails("thingname:alpha-1", "Unable to parse query, invalid field name, field name: thingname");+ }++ @Test+ void wildcardsMatchAnyRunOrOneCharacter() {+ fleet();++ assertEquals(List.of("alpha-1", "alpha-2"), names("thingName:alpha*"));+ assertEquals(List.of("alpha-1", "alpha-2"), names("thingName:ALPHA-?"));+ assertEquals(List.of("beta-1"), names("thingName:b?ta-1"));+ assertEquals(List.of("gamma"), names("thingName:*a"));+ assertEquals(List.of("alpha-1", "alpha-2"), names("attributes.provider:a*e"));+ assertEquals(List.of(), names("thingName:alpha?"));+ assertEquals(List.of(), names("thingName:\"alpha*\""));+ assertEquals(List.of(), names("thingName:alpha\\*"));+ }++ @Test+ void existenceAndMatchAllQueries() {+ fleet();++ assertEquals(List.of("alpha-1", "beta-1"), names("thingTypeName:*"));+ assertEquals(List.of("alpha-1", "alpha-2"), names("attributes.site:*"));+ assertEquals(List.of("alpha-1", "alpha-2"), names("thingGroupNames:*"));+ assertEquals(List.of("alpha-1", "alpha-2", "beta-1", "gamma"), names("*"));+ }++ @Test+ void booleanOperatorsFollowTheMeasuredAwsPrecedence() {+ fleet();++ // AND binds tighter than OR.+ assertEquals(List.of("alpha-2", "gamma"),+ names("thingName:gamma OR attributes.provider:acme AND attributes.site:south"));+ assertEquals(List.of("alpha-2", "gamma"),+ names("attributes.provider:acme AND attributes.site:south OR thingName:gamma"));+ // Whitespace is an AND that binds looser than OR: (gamma OR acme) AND south.+ assertEquals(List.of("alpha-2"), names("thingName:gamma OR attributes.provider:acme attributes.site:south"));+ assertEquals(List.of("alpha-1", "alpha-2"), names("thingName:alpha* attributes.provider:acme"));+ assertEquals(List.of("alpha-2", "gamma"),+ names("(thingName:gamma OR attributes.provider:acme) AND NOT thingTypeName:sensor"));+ // NOT applies to the term after it only.+ assertEquals(List.of("beta-1"), names("NOT attributes.provider:acme AND thingTypeName:sensor"));+ assertEquals(List.of("alpha-2", "gamma"), names("NOT thingTypeName:sensor"));+ assertEquals(List.of("alpha-2", "gamma"), names("-thingTypeName:sensor"));+ assertEquals(List.of("alpha-2"), names("thingName:alpha* -thingTypeName:sensor"));+ assertEquals(List.of("alpha-2"), names("thingName:alpha* - thingTypeName:sensor"));+ assertEquals(List.of("alpha-1", "alpha-2", "gamma"), names("-(thingTypeName:sensor AND attributes.provider:other)"));+ // The symbolic forms AWS accepts.+ assertEquals(List.of("alpha-2", "gamma"), names("!thingTypeName:sensor"));+ assertEquals(List.of("alpha-1"), names("thingTypeName:sensor && attributes.site:north"));+ assertEquals(List.of("beta-1", "gamma"), names("thingName:gamma || attributes.provider:other"));+ }++ @Test+ void documentsCarryOnlyTheMembersAThingHas() {+ fleet();++ assertEquals(json("{\"thingName\": \"alpha-1\", \"thingId\": \""+ + iot.service.describeThing("alpha-1", REGION).getThingId() + "\", \"thingTypeName\": \"sensor\", "+ + "\"thingGroupNames\": [\"north-group\"], \"attributes\": {\"site\": \"North\", \"provider\": \"acme\"}}"),+ document("alpha-1"));+ assertEquals(json("[\"north-group\", \"south-group\"]"), document("alpha-2").get("thingGroupNames"));+ assertEquals(json("{\"thingName\": \"gamma\", \"thingId\": \""+ + iot.service.describeThing("gamma", REGION).getThingId() + "\"}"), document("gamma"));+ }++ @Test+ void connectivityFieldsAreRefusedLikeShadowAndDeviceDefenderFields() {+ fleet("{\"thingIndexingMode\": \"REGISTRY\", \"thingConnectivityIndexingMode\": \"STATUS\"}");++ assertUnsupported("connectivity.connected:true", "the field connectivity.connected");+ assertUnsupported("connectivity.clientId:alpha-1", "the field connectivity.clientId");+ assertUnsupported("connectivity.disconnectReason:CONNECTION_LOST", "the field connectivity.disconnectReason");+ assertUnsupported("thingName:gamma OR connectivity.connected:false", "the field connectivity.connected");+ }++ @Test+ void documentsCarryNoConnectivityWhileConnectivityIsIndexed() {+ fleet("{\"thingIndexingMode\": \"REGISTRY\", \"thingConnectivityIndexingMode\": \"STATUS\"}");++ assertFalse(document("alpha-1").has("connectivity"));+ assertEquals(json("{\"thingName\": \"gamma\", \"thingId\": \""+ + iot.service.describeThing("gamma", REGION).getThingId() + "\"}"), document("gamma"));+ }++ @Test+ void connectivityQueriesNeedConnectivityIndexing() {+ fleet();++ assertSearchFails(query("connectivity.connected:true"), "InvalidRequestException", 400, CONNECTIVITY_NOT_ENABLED);+ assertSearchFails(query("thingName:gamma OR connectivity.clientId:gamma"), "InvalidRequestException", 400,+ CONNECTIVITY_NOT_ENABLED);+ assertFalse(document("alpha-1").has("connectivity"));+ }++ @Test+ void shadowAndDeviceDefenderQueriesNeedTheirIndexing() {+ fleet();++ assertSearchFails(query("shadow.reported.x:1"), "InvalidRequestException", 400, "Query includes one or more "+ + "constraints for Shadow attribute, but Shadow indexing is not enabled for AWS_Things index");+ assertSearchFails(query("deviceDefender.violationCount:1"), "InvalidRequestException", 400,+ "Query includes one or more constraints for Devicedefender attribute, but Devicedefender indexing "+ + "is not enabled for AWS_Things index");+ }++ @Test+ void classicAndNamedShadowFieldsFollowTheirOwnIndexing() {+ fleet("""+ {"thingIndexingMode": "REGISTRY", "namedShadowIndexingMode": "ON",+ "filter": {"namedShadowNames": ["building"]}}+ """);++ assertSearchFails(query("shadow.reported.x:1"), "InvalidRequestException", 400, "Query includes one or more "+ + "constraints for Shadow attribute, but Shadow indexing is not enabled for AWS_Things index");+ assertUnsupported("shadow.name.building.reported.type:x", "the field shadow.name.building.reported.type");+ assertQueryFails("shadow.name.other.reported.a:1",+ "Unable to parse query, invalid field name, field name: shadow.name.other.reported.a");+ assertQueryFails("shadow.name.building:x",+ "Unable to parse query, invalid field name, field name: shadow.name.building");+ assertQueryFails("shadow.name:x", "Unable to parse query, invalid field name, field name: shadow.name");++ update("""+ {"thingIndexingConfiguration": {"thingIndexingMode": "REGISTRY_AND_SHADOW"}}+ """);+ assertQueryFails("shadow.name.building.reported.type:x",+ "Unable to parse query, invalid field name, field name: shadow.name.building.reported.type");+ assertUnsupported("shadow.reported.x:1", "the field shadow.reported.x");+ }++ @Test+ void awsValidConstructsFlociDoesNotEvaluateAreRefused() {+ fleet("""+ {"thingIndexingMode": "REGISTRY_AND_SHADOW", "thingConnectivityIndexingMode": "STATUS",+ "deviceDefenderIndexingMode": "VIOLATIONS"}+ """);++ assertUnsupported("attributes.site:[a TO z]", "range queries");+ assertUnsupported("attributes.site:{a TO z}", "range queries");+ assertUnsupported("thingName>a", "comparisons");+ assertUnsupported("attributes.count>=5", "comparisons");+ assertUnsupported("attributes.count<5", "comparisons");+ assertUnsupported("attributes.count<=5", "comparisons");+ assertUnsupported("alpha", "free text terms");+ assertUnsupported("thingName:alpha-1 or thingName:gamma", "free text terms");+ assertUnsupported("\"alpha-1\"", "free text terms");+ assertUnsupported("attributes.site:(north OR south)", "field grouping");+ assertUnsupported("shadow.reported.x:1", "the field shadow.reported.x");+ assertUnsupported("deviceDefender.violationCount:1", "the field deviceDefender.violationCount");+ assertUnsupported("connectivity.timestamp:0", "the field connectivity.timestamp");+ assertUnsupported("connectivity.connected:true", "the field connectivity.connected");+ }++ @Test+ void malformedQueriesAreSyntaxErrors() {+ fleet();++ for (String queryString : List.of("thingName:", "thingName:(a", "thingName:a AND", "AND thingName:a",+ "thingName:a OR", "thingName:a NOT", "NOT", "!", "()", "thingName:a)", "thingName:a -",+ "NOT NOT thingName:a", "NOT -thingName:a", "-NOT thingName:a", "thingName:a AND OR thingName:b",+ "thingName:\"a", "thingName:a\"", " ", "attributes.count:>5")) {+ assertQueryFails(queryString, "Unable to parse query, invalid syntax");+ }+ }++ @Test+ void rangesAreParsedAsAwsParsesThem() {+ fleet();++ for (String queryString : List.of("thingName:[", "thingName:[a", "thingName:[a TO", "thingName:[a TO b",+ "thingName:[a to b]", "thingName:[a TO b TO c]", "thingName:{", "thingName:]", "thingName:[]",+ "thingName:a]", "thingName:[aTOb]", "thingName:[a TO b]]", "thingName:a]b", "thingName:a[b",+ "thingName:a}", "thingName:a{b", "attributes.a[0]:x", "thingName:[TO b]", "thingName:[a TO]",+ "[a TO]", "thingName:[a TO b)", "thingName:[a\tTO\tb]")) {+ assertQueryFails(queryString, "Unable to parse query, invalid syntax");+ }+ assertQueryFails("bogusfield:1 OR thingName:[a", "Unable to parse query, invalid syntax");+ for (String queryString : List.of("thingName:[a b]", "thingName:[a TO b}", "thingName:[* TO b]",+ "thingName:[ a TO b ]", "thingName:[\"a\" TO \"b\"]", "thingName:[\"a]\" TO b]",+ "thingName:[a[ TO b]", "NOT thingName:[a TO b]", "(thingName:[a TO b])",+ "thingName:[a TO b](thingName:c)")) {+ assertUnsupported(queryString, "range queries");+ }+ assertUnsupported("[a TO b]", "free text terms");+ assertEquals(List.of(), names("thingName:a\\]"));+ assertEquals(List.of(), names("thingName:\"a]\""));++ String range = "thingName:[a TO b] ";+ String eight = String.join(" ", Collections.nCopies(8, "thingName:x"));+ assertUnsupported(range.repeat(4) + eight, "range queries");+ assertQueryFails(range.repeat(5) + eight,+ "Unable to parse query, number of terms cannot exceed 12, terms found: 13");+ }++ @Test+ void unknownFieldsAreInvalidFieldNames() {+ fleet();++ for (String field : List.of("bogusfield", "registry.version", "*", "attributes", "attributes.")) {+ assertQueryFails(field + ":1", "Unable to parse query, invalid field name, field name: " + field);+ }+ }++ @Test+ void queryTypesAwsRejectsAreRejectedWithItsMessages() {+ fleet();++ assertQueryFails("+thingName:a", "Unable to parse query, unsupported operator - \"+\"");+ assertQueryFails("thingName:a~", "Unable to parse query, unsupported query type - fuzzy");+ assertQueryFails("thingName:a~2", "Unable to parse query, unsupported query type - fuzzy");+ assertQueryFails("thingName:/a/", "Unable to parse query, unsupported query type - regular expression");+ assertQueryFails("thingName:a^2", "Unable to parse query, unsupported query type - boost");+ }++ @Test+ void escapedFuzzyAndBoostCharactersAreLiteral() {+ fleet();++ assertEquals(List.of(), names("thingName:a\\~b"));+ assertEquals(List.of(), names("attributes.x:a\\^b"));+ assertEquals(List.of(), names("thingName:\"a~b^c\""));+ assertQueryFails("thingName:a~", "Unable to parse query, unsupported query type - fuzzy");+ }++ @Test+ void aQueryStringOfAtMost1000CodePointsIsAccepted() {+ fleet();+ String longest = "thingName:" + "\u00e9".repeat(990);+ String tooLong = longest + "\u00e9";++ assertEquals(List.of(), names(longest));+ assertSearchFails(query(tooLong), "InvalidRequestException", 400, "1 validation error detected: Value '"+ + tooLong + "' at 'queryString' failed to satisfy constraint: "+ + "Member must have length less than or equal to 1000");+ assertEquals(List.of("gamma"), names("(".repeat(490) + "thingName:gamma" + ")".repeat(490)));+ }++ @Test+ void aQueryOfAtMost12TermsIsAccepted() {+ fleet();+ String twelve = String.join(" OR ", Collections.nCopies(12, "thingName:gamma"));+ String tooMany = "Unable to parse query, number of terms cannot exceed 12, terms found: 13";++ assertEquals(List.of("gamma"), names(twelve));+ assertEquals(List.of("alpha-1", "alpha-2", "beta-1", "gamma"),+ names(String.join(" OR ", Collections.nCopies(7, "NOT thingName:z"))));+ assertQueryFails(twelve + " OR thingName:gamma", tooMany);+ assertQueryFails("* " + twelve, tooMany);+ assertQueryFails("(" + twelve + ") AND NOT thingName:z", tooMany);+ assertQueryFails("bogusfield:1 OR " + twelve, tooMany);+ assertQueryFails(twelve + " OR thingName:*a*b*c*", tooMany);+ }++ @Test+ void syntaxErrorsWinOverTheTermLimitAndOtherQueryErrors() {+ fleet();++ for (String queryString : List.of("thingName:z OR ".repeat(12) + "thingName:", "bogusfield:1 OR thingName:",+ "thingName:a~ OR thingName:", "thingName:*a*b*c* OR thingName:",+ "attributes.site:[a TO b] OR thingName:", "+thingName:a OR thingName:", "thingName:?a OR thingName:",+ "bogusfield:1 OR thingName:<a", "bogusfield:1 OR (thingName:a", "bogusfield:1 AND", "bogusfield:",+ "deviceDefender.violationCount:1 OR thingName:")) {+ assertQueryFails(queryString, "Unable to parse query, invalid syntax");+ }+ }++ @Test+ void theTermLimitWinsOverAFieldWhoseIndexingIsOff() {+ fleet();+ List<String> twelve = IntStream.rangeClosed(1, 12).mapToObj(i -> "thingName:z" + i).toList();++ assertQueryFails("deviceDefender.violationCount:1 OR " + String.join(" OR ", twelve) + " OR thingName:y",+ "Unable to parse query, number of terms cannot exceed 12, terms found: 14");+ }++ @Test+ void fieldGroupValuesCountAsTermsButTheFieldDoesNot() {+ fleet();+ List<String> values = List.of("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m");+ String tooMany = "Unable to parse query, number of terms cannot exceed 12, terms found: 13";++ assertQueryFails("thingName:(" + String.join(" OR ", values) + ")", tooMany);+ assertQueryFails("thingName:(" + String.join(" OR ", values.subList(0, 12)) + ") OR thingName:x", tooMany);+ assertUnsupported("thingName:(" + String.join(" OR ", values.subList(0, 11)) + ") OR thingName:x",+ "field grouping");+ for (String queryString : List.of("thingName:()", "thingName:(a OR)", "thingName:(a OR b")) {+ assertQueryFails(queryString, "Unable to parse query, invalid syntax");+ }+ }++ @Test+ void aFieldGroupChecksItsFieldFirst() {+ fleet();+ String invalidFieldName = "Unable to parse query, invalid field name, field name: ";++ assertQueryFails("bogusfield:(a OR b)", invalidFieldName + "bogusfield");+ assertQueryFails("registry.version:(1 OR 2)", invalidFieldName + "registry.version");+ assertQueryFails("attributes:(a OR b)", invalidFieldName + "attributes");+ assertQueryFails("bogusfield:(a OR b) OR deviceDefender.violationCount:1", invalidFieldName + "bogusfield");+ assertQueryFails("attributes.a[0]:(x OR y)", "Unable to parse query, invalid syntax");+ assertQueryFails("bogusfield:(a OR b) OR thingName:", "Unable to parse query, invalid syntax");+ assertQueryFails("bogusfield:(a OR b OR c OR d OR e OR f OR g OR h OR i OR j OR k OR l OR m)",+ "Unable to parse query, number of terms cannot exceed 12, terms found: 13");+ assertSearchFails(query("connectivity.connected:(true OR false)"), "InvalidRequestException", 400,+ CONNECTIVITY_NOT_ENABLED);+ assertSearchFails(query("deviceDefender.violationCount:(1 OR 2)"), "InvalidRequestException", 400,+ "Query includes one or more constraints for Devicedefender attribute, but Devicedefender indexing "+ + "is not enabled for AWS_Things index");+ assertUnsupported("thingName:(a OR b)", "field grouping");+ }++ @Test+ void aValueTakesAtMostTwoUnescapedWildcards() {+ fleet();++ assertQueryFails("thingName:*a*b*", "Unable to parse query, invalid value - too many wildcards (*). "+ + "The maximum allowed per term is 2, field name: thingName, value: *a*b*");+ assertQueryFails("attributes.x:*a*b*c", "Unable to parse query, invalid value - too many wildcards (*). "+ + "The maximum allowed per term is 2, field name: attributes.x, value: *a*b*c");+ assertEquals(List.of("alpha-1", "alpha-2", "beta-1", "gamma"), names("thingName:*a*"));+ assertEquals(List.of(), names("thingName:*a\\*b*"));+ assertEquals(List.of(), names("thingName:\"*a*b*c*\""));+ assertEquals(List.of(), names("thingName:a\\*b\\*c\\*d"));+ }++ @Test+ void aValueCannotStartWithAQuestionMark() {+ fleet();++ assertQueryFails("thingName:?a", "Unable to parse query, invalid value - wildcard character (?) is not "+ + "allowed at the start of the value, field name: thingName, value: ?a");+ assertEquals(List.of(), names("thingName:a?b"));+ assertEquals(List.of(), names("thingName:a?b?c?d?e?f"));+ }++ @Test+ void requestsAreValidatedInTheMeasuredAwsOrder() {+ assertSearchFails(mapper.createObjectNode(), "InvalidRequestException", 400, "1 validation error detected: "+ + "Value null at 'queryString' failed to satisfy constraint: Member must not be null");+ assertSearchFails(query(""), "InvalidRequestException", 400, "1 validation error detected: Value '' at "+ + "'queryString' failed to satisfy constraint: Member must have length greater than or equal to 1");+ assertSearchFails(query("*").put("maxResults", 0), "InvalidRequestException", 400,+ "1 validation error detected: Value '0' at 'maxResults' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 1");+ assertSearchFails(query("*").put("indexName", ""), "InvalidRequestException", 400,+ "indexName cannot be empty.");+ assertSearchFails(query("*").put("indexName", "Nope").put("queryVersion", "bogus"), "InvalidRequestException",+ 400, "Invalid queryVersion. Expected one of: [2017-09-30]");+ assertSearchFails(query("*").put("indexName", "Nope"), "InvalidRequestException", 400,+ "Unrecognized indexName Nope");+ assertSearchFails(query("bogusfield:1"), "ResourceNotFoundException", 404,+ "Index AWS_Things does not exist. Please enable index by calling UpdateIndexingConfiguration");+ assertSearchFails(query("*").put("indexName", "AWS_ThingGroups"), "ResourceNotFoundException", 404,+ "Index AWS_ThingGroups does not exist. Please enable index by calling UpdateIndexingConfiguration");++ update("""+ {"thingIndexingConfiguration": {"thingIndexingMode": "REGISTRY"},+ "thingGroupIndexingConfiguration": {"thingGroupIndexingMode": "ON"}}+ """);+ assertSearchFails(query("*").put("indexName", "AWS_ThingGroups"), "InvalidRequestException", 400,+ "Floci does not support searching AWS_ThingGroups");+ assertEquals(List.of(), search(query("*").put("indexName", "AWS_Things").put("queryVersion", "2017-09-30"))+ .items());+ }++ @Test+ void pagesCarryATokenUntilTheLastPage() {+ fleet();++ IotService.Page<ObjectNode> first = search(query("thingName:alpha*").put("maxResults", 1));+ assertEquals("alpha-1", first.items().get(0).path("thingName").asText());+ assertEquals(1, first.items().size());+ assertNotNull(first.nextToken());+ IotService.Page<ObjectNode> second =+ search(query("thingName:alpha*").put("maxResults", 1).put("nextToken", first.nextToken()));+ assertEquals("alpha-2", second.items().get(0).path("thingName").asText());+ assertNull(second.nextToken());+ assertNull(search(query("thingName:alpha*")).nextToken());++ assertSearchFails(query("*").put("nextToken", "bogus"), "InvalidRequestException", 400, "Invalid nextToken");+ assertSearchFails(query("bogusfield:1").put("nextToken", "bogus"), "InvalidQueryException", 400,+ "Unable to parse query, invalid field name, field name: bogusfield, query string: bogusfield:1");+ }++ @Test+ void aHugeMaxResultsReturnsTheRemainingThingsWithoutAToken() {+ fleet();+ String token = search(query("*").put("maxResults", 1)).nextToken();++ IotService.Page<ObjectNode> rest =+ search(query("*").put("maxResults", Integer.MAX_VALUE).put("nextToken", token));++ assertEquals(List.of("alpha-2", "beta-1", "gamma"), thingNames(rest));+ assertNull(rest.nextToken());+ }++ @Test+ void aPageTokenContinuesAfterTheLastThingReturnedWhenEarlierThingsAreDeleted() {+ fleet();+ IotService.Page<ObjectNode> first = search(query("thingName:alpha*").put("maxResults", 1));+ assertEquals(List.of("alpha-1"), thingNames(first));+ assertNotNull(first.nextToken());++ iot.service.deleteThing("alpha-1", REGION);+ IotService.Page<ObjectNode> second =+ search(query("thingName:alpha*").put("maxResults", 1).put("nextToken", first.nextToken()));++ assertEquals(List.of("alpha-2"), thingNames(second));+ assertNull(second.nextToken());+ }++ @Test+ void aPageTokenOnlyContinuesTheRequestThatIssuedIt() {+ fleet();+ String token = search(query("thingName:alpha*").put("maxResults", 1)).nextToken();++ assertEquals(List.of("alpha-2"),+ thingNames(search(query("thingName:alpha*").put("maxResults", 5).put("nextToken", token))));+ assertEquals(List.of("alpha-2"), thingNames(search(query("thingName:alpha*").put("nextToken", token))));+ String differentParameters = "nextToken was from a request with different parameters";+ assertSearchFails(query("thingName:alpha* ").put("nextToken", token), "InvalidRequestException", 400,+ differentParameters);+ assertSearchFails(query("*").put("nextToken", token), "InvalidRequestException", 400, differentParameters);+ assertSearchFails(query("thingName:alpha*").put("indexName", "AWS_Things").put("nextToken", token),+ "InvalidRequestException", 400, differentParameters);+ assertSearchFails(query("thingName:alpha*").put("queryVersion", "2017-09-30").put("nextToken", token),+ "InvalidRequestException", 400, differentParameters);++ String explicit = search(query("thingName:alpha*").put("indexName", "AWS_Things")+ .put("queryVersion", "2017-09-30").put("maxResults", 1)).nextToken();+ assertEquals(List.of("alpha-2"), thingNames(search(query("thingName:alpha*").put("indexName", "AWS_Things")+ .put("queryVersion", "2017-09-30").put("nextToken", explicit))));+ assertSearchFails(query("thingName:alpha*").put("nextToken", explicit), "InvalidRequestException", 400,+ differentParameters);++ assertSearchFails(query("bogusfield:1").put("nextToken", token), "InvalidQueryException", 400,+ "Unable to parse query, invalid field name, field name: bogusfield, query string: bogusfield:1");+ }++ @Test+ void aTokenThisOperationCouldNotHaveIssuedIsInvalid() {+ fleet();+ String bareThingName = Base64.getUrlEncoder().withoutPadding()+ .encodeToString("alpha-1".getBytes(StandardCharsets.UTF_8));++ for (String token : List.of("YQ", "", "AAAA", bareThingName)) {+ assertSearchFails(query("*").put("nextToken", token), "InvalidRequestException", 400,+ "Invalid nextToken");+ }++ String issued = search(query("thingName:alpha*").put("maxResults", 1)).nextToken();+ String decoded = new String(Base64.getUrlDecoder().decode(issued), StandardCharsets.UTF_8);+ String edited = Base64.getUrlEncoder().withoutPadding()+ .encodeToString((decoded.substring(0, decoded.length() - 1) + "0").getBytes(StandardCharsets.UTF_8));+ String shortened = Base64.getUrlEncoder().withoutPadding()+ .encodeToString(decoded.substring(0, decoded.length() - 1).getBytes(StandardCharsets.UTF_8));+ String truncated = issued.substring(0, issued.length() - 2);+ for (String token : List.of(edited, shortened, truncated)) {+ assertSearchFails(query("thingName:alpha*").put("nextToken", token), "InvalidRequestException", 400,+ "Invalid nextToken");+ }+ assertEquals(List.of("alpha-2"), thingNames(search(query("thingName:alpha*").put("nextToken", issued))));+ }++ @Test+ void maxResultsIsAnyJsonNumberOfAtLeastOne() {+ fleet();++ assertEquals(List.of("alpha-1"), thingNames(search(query("*").put("maxResults", 1.5))));+ IotService.Page<ObjectNode> all = search(query("*").put("maxResults", 2147483648L));+ assertEquals(List.of("alpha-1", "alpha-2", "beta-1", "gamma"), thingNames(all));+ assertNull(all.nextToken());+ assertEquals(4, search(query("*").put("maxResults", 500)).items().size());+ assertSearchFails(query("*").put("maxResults", "5"), "SerializationException", 400,+ "maxResults must be a JSON number");+ assertSearchFails(query("*").put("maxResults", true), "SerializationException", 400,+ "maxResults must be a JSON number");+ }++ @Test+ void maxResultsIsTruncatedAndSaturatedAtTheIntRange() {+ fleet();++ IotService.Page<ObjectNode> all = search(query("*").put("maxResults", new BigInteger("9223372036854775808")));+ assertEquals(List.of("alpha-1", "alpha-2", "beta-1", "gamma"), thingNames(all));+ assertNull(all.nextToken());+ assertEquals(4, search(query("*").put("maxResults", new BigDecimal("1e30"))).items().size());+ assertSearchFails(query("*").put("maxResults", new BigInteger("-9223372036854775809")),+ "InvalidRequestException", 400, "1 validation error detected: Value '-2147483648' at 'maxResults' "+ + "failed to satisfy constraint: Member must have value greater than or equal to 1");+ assertSearchFails(query("*").put("maxResults", 0.5), "InvalidRequestException", 400,+ "1 validation error detected: Value '0' at 'maxResults' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 1");+ assertSearchFails(query("*").put("maxResults", -1), "InvalidRequestException", 400,+ "1 validation error detected: Value '-1' at 'maxResults' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 1");+ }++ @Test+ void indexNameMustBeAJsonString() {+ fleet();++ assertSearchFails(query("*").put("indexName", 1), "SerializationException", 400,+ "indexName must be a JSON string");+ assertSearchFails(query("*").put("indexName", true), "SerializationException", 400,+ "indexName must be a JSON string");+ }++ @Test+ void aBodyThatIsNotAJsonObjectIsASerializationError() {+ fleet();++ for (String body : List.of("[]", "\"x\"", "1", "null", "[{\"queryString\":\"*\"}]")) {+ assertSearchBodyFails(body, "request body must be a JSON object");+ }+ }++ @Test+ void memberTypeErrorsComeBeforeEveryOtherError() {+ fleet();++ assertSearchBodyFails("{\"queryString\":\"bogusfield:\",\"nextToken\":1}", "nextToken must be a JSON string");+ assertSearchBodyFails("{\"queryString\":\"*\",\"indexName\":\"\",\"queryVersion\":1}",+ "queryVersion must be a JSON string");+ assertSearchBodyFails("{\"queryString\":\"\",\"nextToken\":true}", "nextToken must be a JSON string");+ assertSearchBodyFails("{\"queryString\":\"*\",\"indexName\":\"nope\",\"nextToken\":1}",+ "nextToken must be a JSON string");+ }++ private void assertSearchBodyFails(String body, String message) {+ AwsException failure = assertThrows(AwsException.class,+ () -> service.searchIndex(json(body), REGION), body);+ assertEquals("SerializationException", failure.getErrorCode(), body);+ assertEquals(400, failure.getHttpStatus(), body);+ assertEquals(message, failure.getMessage(), body);+ }++ private static List<String> thingNames(IotService.Page<ObjectNode> page) {+ return page.items().stream().map(thing -> thing.path("thingName").asText()).toList();+ }++ @Test+ void searchSeesOnlyTheCallersRegion() {+ fleet();+ iot.service.createThing("alpha-9", Map.of("provider", "acme"), null, "eu-west-1");++ assertEquals(List.of("alpha-1", "alpha-2"), names("attributes.provider:acme"));+ assertThrows(AwsException.class, () -> service.searchIndex(query("*"), "eu-west-1"));++ service.updateIndexingConfiguration(json("""+ {"thingIndexingConfiguration": {"thingIndexingMode": "REGISTRY"}}+ """), "eu-west-1");+ List<ObjectNode> other = service.searchIndex(query("attributes.provider:acme"), "eu-west-1").items();+ assertEquals(1, other.size());+ assertEquals("alpha-9", other.get(0).path("thingName").asText());+ }}