Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix: apply S3 Vectors metadata filters (#2417)

Matt Van Hornbd8d99b7575b
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3VectorsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3VectorsTest.java
index cd3395e88d..4ba9b77300 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3VectorsTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3VectorsTest.java
@@ -128,12 +128,16 @@ class S3VectorsTest {
PutInputVector.builder()
.key("v1")
.data(VectorData.builder().float32(1.0f, 0.0f, 0.0f).build())
- .metadata(Document.fromMap(Map.of("label", Document.fromString("first"))))
+ .metadata(Document.fromMap(Map.of(
+ "label", Document.fromString("first"),
+ "tenant", Document.fromString("tenant-a"))))
.build(),
PutInputVector.builder()
.key("v2")
.data(VectorData.builder().float32(0.0f, 1.0f, 0.0f).build())
- .metadata(Document.fromMap(Map.of("label", Document.fromString("second"))))
+ .metadata(Document.fromMap(Map.of(
+ "label", Document.fromString("second"),
+ "tenant", Document.fromString("tenant-b"))))
.build()
)
.build()
@@ -185,6 +189,26 @@ class S3VectorsTest {
@Test
@Order(10)
+ void queryVectorsWithMetadataFilter() {
+ QueryVectorsResponse response = client.queryVectors(
+ QueryVectorsRequest.builder()
+ .vectorBucketName(bucketName)
+ .indexName(indexName)
+ .queryVector(VectorData.builder().float32(1.0f, 0.0f, 0.0f).build())
+ .topK(1)
+ .filter(Document.fromMap(Map.of("tenant", Document.fromString("tenant-b"))))
+ .returnMetadata(true)
+ .build()
+ );
+
+ assertThat(response.vectors()).singleElement().satisfies(result -> {
+ assertThat(result.key()).isEqualTo("v2");
+ assertThat(result.metadata().asMap().get("tenant").asString()).isEqualTo("tenant-b");
+ });
+ }
+
+ @Test
+ @Order(11)
void deleteVectors() {
client.deleteVectors(
DeleteVectorsRequest.builder()
@@ -206,7 +230,7 @@ class S3VectorsTest {
}
@Test
- @Order(11)
+ @Order(12)
void testConflictException() {
assertThatThrownBy(() -> client.createVectorBucket(
CreateVectorBucketRequest.builder()
@@ -216,7 +240,7 @@ class S3VectorsTest {
}
@Test
- @Order(12)
+ @Order(13)
void testNotFoundException() {
assertThatThrownBy(() -> client.getVectorBucket(
GetVectorBucketRequest.builder()
@@ -226,7 +250,7 @@ class S3VectorsTest {
}
@Test
- @Order(13)
+ @Order(14)
void testValidationException() {
assertThatThrownBy(() -> client.createIndex(
CreateIndexRequest.builder()
diff --git a/docs/services/s3vectors.md b/docs/services/s3vectors.md
index 372c60fa28..adeb8735e1 100644
--- a/docs/services/s3vectors.md
+++ b/docs/services/s3vectors.md
@@ -34,3 +34,20 @@ aws s3vectors query-vectors --vector-bucket-name my-vectors --index-name embeddi
--query-vector '{"float32":[0.1,0.2,0.3,0.4]}' --top-k 1 \
--endpoint-url http://localhost:4566
```
+
+## Metadata filters
+
+`QueryVectors` supports metadata filters with `$eq`, `$ne`, `$gt`, `$gte`,
+`$lt`, `$lte`, `$in`, `$nin`, and `$exists` field predicates. `$and` and `$or`
+recursively combine filters, while multiple clauses in the same object are
+implicitly combined with AND. A scalar field value is shorthand for `$eq`.
+
+```bash
+aws s3vectors query-vectors --vector-bucket-name my-vectors --index-name embeddings \
+ --query-vector '{"float32":[0.1,0.2,0.3,0.4]}' --top-k 5 \
+ --filter '{"$and":[{"tenant":"tenant-a"},{"year":{"$gte":2024}}]}' \
+ --endpoint-url http://localhost:4566
+```
+
+Filters are evaluated before similarity ranking and `topK` truncation, so a
+nearer vector that does not match the filter cannot consume a result slot.
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java
index 7731a3eb1c..5f03504ae5 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java
@@ -429,6 +429,7 @@ public class S3VectorsController {
request.vectorBucketName(),
request.indexName(),
queryVector,
+ request.filter(),
request.topK() > 0 ? request.topK() : 10,
region
);
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsMetadataFilter.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsMetadataFilter.java
new file mode 100644
index 0000000000..02c7205b07
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsMetadataFilter.java
@@ -0,0 +1,264 @@
+package io.github.hectorvent.floci.services.s3vectors;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+final class S3VectorsMetadataFilter {
+
+ private static final S3VectorsMetadataFilter MATCH_ALL = new S3VectorsMetadataFilter(metadata -> true);
+
+ private final FilterNode root;
+
+ private S3VectorsMetadataFilter(FilterNode root) {
+ this.root = root;
+ }
+
+ static S3VectorsMetadataFilter compile(Object filter) {
+ if (filter == null) {
+ return MATCH_ALL;
+ }
+ return new S3VectorsMetadataFilter(compileFilter(filter));
+ }
+
+ boolean matches(Map<String, Object> metadata) {
+ return root.matches(metadata != null ? metadata : Map.of());
+ }
+
+ private static FilterNode compileFilter(Object filter) {
+ if (!(filter instanceof Map<?, ?> clauses) || clauses.isEmpty()) {
+ throw validation("A metadata filter must be a non-empty object.");
+ }
+
+ List<FilterNode> nodes = new ArrayList<>();
+ for (Map.Entry<?, ?> clause : clauses.entrySet()) {
+ if (!(clause.getKey() instanceof String key) || key.isEmpty()) {
+ throw validation("Metadata filter keys must be non-empty strings.");
+ }
+ Operator operator = Operator.fromWireValue(key);
+ if (operator == Operator.AND || operator == Operator.OR) {
+ nodes.add(compileLogical(operator, clause.getValue()));
+ } else if (key.startsWith("$")) {
+ throw validation("Unsupported metadata filter operator: " + key);
+ } else {
+ nodes.add(compileField(key, clause.getValue()));
+ }
+ }
+ return all(nodes);
+ }
+
+ private static FilterNode compileLogical(Operator operator, Object operand) {
+ if (!(operand instanceof List<?> filters) || filters.isEmpty()) {
+ throw validation(operator.wireValue + " requires a non-empty array of filters.");
+ }
+ List<FilterNode> nodes = new ArrayList<>();
+ for (Object filter : filters) {
+ nodes.add(compileFilter(filter));
+ }
+ return operator == Operator.AND ? all(nodes) : any(nodes);
+ }
+
+ private static FilterNode compileField(String field, Object predicate) {
+ if (!(predicate instanceof Map<?, ?> operators)) {
+ return comparison(field, Operator.EQ, compileValueMatcher(predicate, Operator.EQ));
+ }
+ if (operators.isEmpty()) {
+ throw validation("The predicate for metadata field " + field + " must not be empty.");
+ }
+
+ List<FilterNode> nodes = new ArrayList<>();
+ for (Map.Entry<?, ?> entry : operators.entrySet()) {
+ if (!(entry.getKey() instanceof String wireValue)) {
+ throw validation("Metadata predicate operators must be strings.");
+ }
+ Operator operator = Operator.fromWireValue(wireValue);
+ if (operator == null) {
+ throw validation("Unsupported metadata filter operator: " + wireValue);
+ }
+ nodes.add(compileOperator(field, operator, entry.getValue()));
+ }
+ return all(nodes);
+ }
+
+ private static FilterNode compileOperator(String field, Operator operator, Object operand) {
+ return switch (operator) {
+ case EQ, NE -> comparison(field, operator, compileValueMatcher(operand, operator));
+ case GT, GTE, LT, LTE -> {
+ if (!(operand instanceof Number number)) {
+ throw validation(operator.wireValue + " requires a numeric operand.");
+ }
+ BigDecimal expected = decimal(number, operator.wireValue + " operand");
+ yield orderedComparison(field, operator, expected);
+ }
+ case IN, NIN -> compileMembership(field, operator, operand);
+ case EXISTS -> {
+ if (!(operand instanceof Boolean expected)) {
+ throw validation("$exists requires a boolean operand.");
+ }
+ yield metadata -> metadata.containsKey(field) == expected;
+ }
+ case AND, OR -> throw validation("Unsupported field predicate operator: " + operator.wireValue);
+ };
+ }
+
+ private static FilterNode comparison(String field, Operator operator, ValueMatcher expected) {
+ if (operator == Operator.EQ) {
+ return metadata -> metadata.containsKey(field) && valueMatches(metadata.get(field), expected);
+ }
+ return metadata -> !metadata.containsKey(field) || !valueMatches(metadata.get(field), expected);
+ }
+
+ private static FilterNode orderedComparison(String field, Operator operator, BigDecimal expected) {
+ return switch (operator) {
+ case GT -> metadata -> compareNumber(metadata, field, expected, comparison -> comparison > 0);
+ case GTE -> metadata -> compareNumber(metadata, field, expected, comparison -> comparison >= 0);
+ case LT -> metadata -> compareNumber(metadata, field, expected, comparison -> comparison < 0);
+ case LTE -> metadata -> compareNumber(metadata, field, expected, comparison -> comparison <= 0);
+ case EQ, NE, IN, NIN, EXISTS, AND, OR ->
+ throw new IllegalArgumentException("Not an ordered comparison: " + operator.wireValue);
+ };
+ }
+
+ private static boolean compareNumber(Map<String, Object> metadata, String field, BigDecimal expected,
+ ComparisonMatcher matcher) {
+ if (!metadata.containsKey(field)) {
+ return false;
+ }
+ Object actual = metadata.get(field);
+ if (!(actual instanceof Number number)) {
+ return false;
+ }
+ return matcher.matches(decimal(number, "metadata field " + field).compareTo(expected));
+ }
+
+ private static FilterNode compileMembership(String field, Operator operator, Object operand) {
+ if (!(operand instanceof List<?> candidates) || candidates.isEmpty()) {
+ throw validation(operator.wireValue + " requires a non-empty array of primitive values.");
+ }
+ List<ValueMatcher> matchers = new ArrayList<>();
+ for (Object candidate : candidates) {
+ matchers.add(compileValueMatcher(candidate, operator));
+ }
+
+ boolean includeMatches = operator == Operator.IN;
+ return metadata -> {
+ if (!metadata.containsKey(field)) {
+ return !includeMatches;
+ }
+ Object actual = metadata.get(field);
+ boolean matched = false;
+ for (ValueMatcher matcher : matchers) {
+ if (valueMatches(actual, matcher)) {
+ matched = true;
+ break;
+ }
+ }
+ return includeMatches == matched;
+ };
+ }
+
+ private static boolean valueMatches(Object actual, ValueMatcher expected) {
+ if (actual instanceof List<?> values) {
+ for (Object value : values) {
+ if (expected.matches(value)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ return expected.matches(actual);
+ }
+
+ private static ValueMatcher compileValueMatcher(Object value, Operator operator) {
+ if (!(value instanceof String) && !(value instanceof Number) && !(value instanceof Boolean)) {
+ throw validation(operator.wireValue + " requires a string, number, or boolean operand.");
+ }
+ if (value instanceof Number number) {
+ BigDecimal expected = decimal(number, operator.wireValue + " operand");
+ return actual -> actual instanceof Number actualNumber
+ && decimal(actualNumber, "metadata value").compareTo(expected) == 0;
+ }
+ return actual -> actual != null && actual.getClass() == value.getClass() && actual.equals(value);
+ }
+
+ private static FilterNode all(List<FilterNode> nodes) {
+ return metadata -> {
+ for (FilterNode node : nodes) {
+ if (!node.matches(metadata)) {
+ return false;
+ }
+ }
+ return true;
+ };
+ }
+
+ private static FilterNode any(List<FilterNode> nodes) {
+ return metadata -> {
+ for (FilterNode node : nodes) {
+ if (node.matches(metadata)) {
+ return true;
+ }
+ }
+ return false;
+ };
+ }
+
+ private static BigDecimal decimal(Number number, String description) {
+ try {
+ return new BigDecimal(number.toString());
+ } catch (NumberFormatException exception) {
+ throw validation(description + " must be a finite number.");
+ }
+ }
+
+ private static AwsException validation(String message) {
+ return new AwsException("ValidationException", message, 400);
+ }
+
+ @FunctionalInterface
+ private interface FilterNode {
+ boolean matches(Map<String, Object> metadata);
+ }
+
+ @FunctionalInterface
+ private interface ValueMatcher {
+ boolean matches(Object actual);
+ }
+
+ @FunctionalInterface
+ private interface ComparisonMatcher {
+ boolean matches(int comparison);
+ }
+
+ private enum Operator {
+ EQ("$eq"),
+ NE("$ne"),
+ GT("$gt"),
+ GTE("$gte"),
+ LT("$lt"),
+ LTE("$lte"),
+ IN("$in"),
+ NIN("$nin"),
+ EXISTS("$exists"),
+ AND("$and"),
+ OR("$or");
+
+ private final String wireValue;
+
+ Operator(String wireValue) {
+ this.wireValue = wireValue;
+ }
+
+ private static Operator fromWireValue(String wireValue) {
+ for (Operator operator : values()) {
+ if (operator.wireValue.equals(wireValue)) {
+ return operator;
+ }
+ }
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java
index ee39639642..75c712d544 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java
@@ -263,17 +263,22 @@ public class S3VectorsService {
LOG.infov("Deleted {0} vectors from index {1}", keys.size(), indexName);
}
- public List<QueryResult> queryVectors(String bucketName, String indexName, List<Float> queryVector, int topK, String region) {
+ public List<QueryResult> queryVectors(String bucketName, String indexName, List<Float> queryVector,
+ Object filter, int topK, String region) {
VectorIndex index = getIndex(bucketName, indexName, region);
if (queryVector.size() != index.getDimension()) {
throw new AwsException("ValidationException",
"Query vector dimension " + queryVector.size() + " does not match index dimension " + index.getDimension(), 400);
}
+ S3VectorsMetadataFilter metadataFilter = S3VectorsMetadataFilter.compile(filter);
String metric = index.getDistanceMetric() != null ? index.getDistanceMetric().toLowerCase() : "cosine";
List<QueryResult> results = new ArrayList<>();
for (VectorData v : index.getVectors().values()) {
+ if (!metadataFilter.matches(v.getMetadata())) {
+ continue;
+ }
double distance = 0.0;
switch (metric) {
case "euclidean":
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsIntegrationTest.java
index d7428dd71f..415177d050 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsIntegrationTest.java
@@ -150,7 +150,8 @@ class S3VectorsIntegrationTest {
"float32": [1.0, 0.0, 0.0]
},
"metadata": {
- "label": "first"
+ "label": "first",
+ "tenant": "tenant-a"
}
},
{
@@ -159,7 +160,8 @@ class S3VectorsIntegrationTest {
"float32": [0.0, 1.0, 0.0]
},
"metadata": {
- "label": "second"
+ "label": "second",
+ "tenant": "tenant-b"
}
}
]
@@ -225,6 +227,87 @@ class S3VectorsIntegrationTest {
@Test
@Order(10)
+ void queryVectorsFiltersByMetadataEquality() {
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ {
+ "vectorBucketName": "%s",
+ "indexName": "%s",
+ "queryVector": {
+ "float32": [1.0, 0.0, 0.0]
+ },
+ "topK": 2,
+ "filter": {
+ "tenant": {
+ "$eq": "tenant-a"
+ }
+ },
+ "returnMetadata": true
+ }
+ """.formatted(BUCKET_NAME, INDEX_NAME))
+ .when()
+ .post("/QueryVectors")
+ .then()
+ .statusCode(200)
+ .body("vectors", hasSize(1))
+ .body("vectors[0].key", equalTo("v1"))
+ .body("vectors[0].metadata.tenant", equalTo("tenant-a"));
+ }
+
+ @Test
+ @Order(11)
+ void queryVectorsReturnsEmptyWhenFilterDoesNotMatch() {
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ {
+ "vectorBucketName": "%s",
+ "indexName": "%s",
+ "queryVector": {
+ "float32": [1.0, 0.0, 0.0]
+ },
+ "topK": 2,
+ "filter": {
+ "tenant": "tenant-c"
+ }
+ }
+ """.formatted(BUCKET_NAME, INDEX_NAME))
+ .when()
+ .post("/QueryVectors")
+ .then()
+ .statusCode(200)
+ .body("vectors", empty());
+ }
+
+ @Test
+ @Order(12)
+ void queryVectorsFiltersBeforeApplyingTopK() {
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ {
+ "vectorBucketName": "%s",
+ "indexName": "%s",
+ "queryVector": {
+ "float32": [1.0, 0.0, 0.0]
+ },
+ "topK": 1,
+ "filter": {
+ "tenant": "tenant-b"
+ }
+ }
+ """.formatted(BUCKET_NAME, INDEX_NAME))
+ .when()
+ .post("/QueryVectors")
+ .then()
+ .statusCode(200)
+ .body("vectors", hasSize(1))
+ .body("vectors[0].key", equalTo("v2"));
+ }
+
+ @Test
+ @Order(13)
void queryVectors_withoutReturnDistance_omitsDistance() {
given()
.contentType(JSON_CONTENT_TYPE)
@@ -243,11 +326,12 @@ class S3VectorsIntegrationTest {
.then()
.statusCode(200)
.body("vectors", hasSize(1))
- .body("vectors[0]", not(hasKey("distance")));
+ .body("vectors[0]", not(hasKey("distance")))
+ .body("vectors[0]", not(hasKey("metadata")));
}
@Test
- @Order(11)
+ @Order(14)
void listVectors() {
// Reproduces #2161: ListVectors previously had no route and fell through to the S3
// REST-XML controller, returning an S3 <Error>InvalidArgument</Error> instead of a
@@ -275,7 +359,7 @@ class S3VectorsIntegrationTest {
}
@Test
- @Order(12)
+ @Order(15)
void listVectorsPaginatesWithMaxResultsAndNextToken() {
String firstPageNextToken =
given()
@@ -316,7 +400,7 @@ class S3VectorsIntegrationTest {
}
@Test
- @Order(13)
+ @Order(16)
void deleteVectors() {
given()
.contentType(JSON_CONTENT_TYPE)
@@ -351,7 +435,7 @@ class S3VectorsIntegrationTest {
}
@Test
- @Order(14)
+ @Order(17)
void deleteIndex() {
given()
.contentType(JSON_CONTENT_TYPE)
@@ -382,7 +466,7 @@ class S3VectorsIntegrationTest {
}
@Test
- @Order(15)
+ @Order(18)
void deleteVectorBucket() {
given()
.contentType(JSON_CONTENT_TYPE)
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsMetadataFilterTest.java b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsMetadataFilterTest.java
new file mode 100644
index 0000000000..3295a595f9
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsMetadataFilterTest.java
@@ -0,0 +1,141 @@
+package io.github.hectorvent.floci.services.s3vectors;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import org.junit.jupiter.api.Test;
+
+import java.math.BigDecimal;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class S3VectorsMetadataFilterTest {
+
+ @Test
+ void matchesEqualityAndInequalityAcrossScalarTypes() {
+ Map<String, Object> metadata = Map.of(
+ "name", "documentary",
+ "published", true,
+ "score", 2,
+ "genres", List.of("documentary", "comedy"));
+
+ assertTrue(matches(metadata, Map.of("name", "documentary")));
+ assertTrue(matches(metadata, Map.of("published", Map.of("$eq", true))));
+ assertTrue(matches(metadata, Map.of("score", Map.of("$eq", 2.0))));
+ assertTrue(matches(metadata, Map.of("score", Map.of("$eq", new BigDecimal("2.00")))));
+ assertTrue(matches(metadata, Map.of("genres", Map.of("$eq", "comedy"))));
+ assertTrue(matches(metadata, Map.of("name", Map.of("$ne", "drama"))));
+ assertFalse(matches(metadata, Map.of("score", Map.of("$ne", 2.0))));
+ assertTrue(matches(metadata, Map.of("score", Map.of("$ne", 3.0))));
+
+ assertFalse(matches(metadata, Map.of("name", Map.of("$eq", true))));
+ assertFalse(matches(metadata, Map.of("score", Map.of("$eq", "2"))));
+ assertFalse(matches(metadata, Map.of("published", Map.of("$ne", true))));
+ }
+
+ @Test
+ void matchesOrderedComparisonsAndBoundaries() {
+ Map<String, Object> metadata = Map.of("score", 10);
+
+ assertTrue(matches(metadata, Map.of("score", Map.of("$gt", 9))));
+ assertFalse(matches(metadata, Map.of("score", Map.of("$gt", 10))));
+ assertTrue(matches(metadata, Map.of("score", Map.of("$gte", 10.0))));
+ assertTrue(matches(metadata, Map.of("score", Map.of("$lt", 11))));
+ assertFalse(matches(metadata, Map.of("score", Map.of("$lt", 10))));
+ assertTrue(matches(metadata, Map.of("score", Map.of("$lte", new BigDecimal("10.00")))));
+ assertTrue(matches(metadata, Map.of("score", Map.of("$gte", 10, "$lte", 10))));
+ assertFalse(matches(Map.of("score", "10"), Map.of("score", Map.of("$gt", 9))));
+ }
+
+ @Test
+ void matchesMembershipPredicates() {
+ Map<String, Object> metadata = Map.of(
+ "genre", "documentary",
+ "tags", List.of("history", "nature"));
+
+ assertTrue(matches(metadata, Map.of("genre", Map.of("$in", List.of("drama", "documentary")))));
+ assertFalse(matches(metadata, Map.of("genre", Map.of("$in", List.of("drama", "comedy")))));
+ assertTrue(matches(metadata, Map.of("tags", Map.of("$in", List.of("nature", "science")))));
+ assertTrue(matches(metadata, Map.of("genre", Map.of("$nin", List.of("drama", "comedy")))));
+ assertFalse(matches(metadata, Map.of("tags", Map.of("$nin", List.of("nature", "science")))));
+ }
+
+ @Test
+ void handlesMissingFieldsAndPresentNullValuesExplicitly() {
+ Map<String, Object> metadata = new LinkedHashMap<>();
+ metadata.put("nullable", null);
+
+ assertTrue(matches(metadata, Map.of("nullable", Map.of("$exists", true))));
+ assertFalse(matches(metadata, Map.of("missing", Map.of("$exists", true))));
+ assertTrue(matches(metadata, Map.of("missing", Map.of("$exists", false))));
+ assertFalse(matches(metadata, Map.of("nullable", Map.of("$exists", false))));
+
+ assertFalse(matches(metadata, Map.of("missing", Map.of("$eq", "value"))));
+ assertFalse(matches(metadata, Map.of("missing", Map.of("$gt", 1))));
+ assertFalse(matches(metadata, Map.of("missing", Map.of("$in", List.of("value")))));
+ assertTrue(matches(metadata, Map.of("missing", Map.of("$ne", "value"))));
+ assertTrue(matches(metadata, Map.of("missing", Map.of("$nin", List.of("value")))));
+ assertFalse(matches(metadata, Map.of("nullable", Map.of("$eq", "value"))));
+ assertTrue(matches(metadata, Map.of("nullable", Map.of("$ne", "value"))));
+ assertFalse(matches(metadata, Map.of("nullable", Map.of("$in", List.of("value")))));
+ assertTrue(matches(metadata, Map.of("nullable", Map.of("$nin", List.of("value")))));
+ }
+
+ @Test
+ void recursivelyCombinesLogicalAndFieldClauses() {
+ Map<String, Object> metadata = Map.of(
+ "tenant", "tenant-a",
+ "year", 2024,
+ "published", true);
+
+ Map<String, Object> filter = Map.of(
+ "tenant", "tenant-a",
+ "$and", List.of(
+ Map.of("year", Map.of("$gte", 2020)),
+ Map.of("$or", List.of(
+ Map.of("published", false),
+ Map.of("published", true)))));
+
+ assertTrue(matches(metadata, filter));
+ assertFalse(matches(metadata, Map.of(
+ "tenant", "tenant-a",
+ "year", Map.of("$lt", 2020))));
+ assertFalse(matches(metadata, Map.of("$and", List.of(
+ Map.of("tenant", "tenant-a"),
+ Map.of("published", false)))));
+ assertFalse(matches(metadata, Map.of("$or", List.of(
+ Map.of("tenant", "tenant-b"),
+ Map.of("year", Map.of("$lt", 2020))))));
+ }
+
+ @Test
+ void rejectsMalformedOrUnsupportedFilters() {
+ assertValidation(Map.of());
+ assertValidation(Map.of("$and", Map.of("tenant", "tenant-a")));
+ assertValidation(Map.of("$and", List.of("tenant-a")));
+ assertValidation(Map.of("$or", List.of()));
+ assertValidation(Map.of("tenant", Map.of("$in", "tenant-a")));
+ assertValidation(Map.of("tenant", Map.of("$nin", List.of())));
+ assertValidation(Map.of("tenant", Map.of("$in", List.of(Map.of("nested", true)))));
+ assertValidation(Map.of("tenant", Map.of("$unknown", "tenant-a")));
+ assertValidation(Map.of("tenant", Map.of("$exists", "true")));
+ assertValidation(Map.of("year", Map.of("$gt", "2020")));
+ assertValidation(Map.of("year", Map.of("$eq", List.of(2020))));
+ assertValidation(Map.of("tenant", Map.of()));
+ assertValidation("tenant-a");
+ }
+
+ private boolean matches(Map<String, Object> metadata, Object filter) {
+ return S3VectorsMetadataFilter.compile(filter).matches(metadata);
+ }
+
+ private void assertValidation(Object filter) {
+ AwsException exception = assertThrows(AwsException.class, () -> S3VectorsMetadataFilter.compile(filter));
+ assertEquals("ValidationException", exception.getErrorCode());
+ assertEquals(400, exception.getHttpStatus());
+ }
+}