Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3vectors): preserve resource configuration and serialize writes (#5892)
andrehoong-px06bd8c2207ec
diff --git a/docs/services/s3vectors.md b/docs/services/s3vectors.mdindex adeb8735e1..8281dc691d 100644--- a/docs/services/s3vectors.md+++ b/docs/services/s3vectors.md@@ -51,3 +51,26 @@ aws s3vectors query-vectors --vector-bucket-name my-vectors --index-name embeddiFilters are evaluated before similarity ranking and `topK` truncation, so anearer vector that does not match the filter cannot consume a result slot.++## Resource configuration++Vector buckets and indexes retain their creation times, encryption settings, and+creation-time tags. Get and list responses include creation times. `GetIndex`+returns encryption settings and includes metadata configuration only when+non-filterable keys are configured. Indexes inherit their bucket's encryption+settings unless overridden. These fields use the existing S3 Vectors storage backend.++On first read, older stored resources with missing encryption settings receive the+bucket default, or `AES256` when the bucket has no setting. Missing creation times+are set to the migration time and saved through the configured storage backend;+the original creation time is unavailable. Later reads and restarts retain that value.++Index dimensions must be between 1 and 4096. Metadata configuration accepts up to+10 unique non-filterable keys, each a non-blank string of 1 to 63 characters.+Queries reject filters on these keys, including filters nested in logical expressions.++Writes to the same bucket are serialized across indexes, vectors, and bucket+deletion so concurrent changes do not overwrite each other or restore deleted data.++Encryption settings are stored and returned; Floci does not perform KMS encryption.+Standalone S3 Vectors tagging APIs are not implemented.diff --git a/pom.xml b/pom.xmlindex e448fb2bb2..66f3903112 100644--- a/pom.xml+++ b/pom.xml@@ -330,6 +330,12 @@</dependency><!-- Testing -->+ <dependency>+ <groupId>software.amazon.awssdk</groupId>+ <artifactId>s3vectors</artifactId>+ <version>${aws.sdk.version}</version>+ <scope>test</scope>+ </dependency><dependency><groupId>software.amazon.awssdk</groupId><artifactId>elasticache</artifactId>diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.javaindex 5f03504ae5..1648cbcd7a 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java@@ -19,7 +19,9 @@ import jakarta.ws.rs.core.MediaType;import jakarta.ws.rs.core.Response;import org.jboss.logging.Logger;-import java.util.*;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;import java.util.stream.Collectors;@Path("/")@@ -43,7 +45,8 @@ public class S3VectorsController {@RegisterForReflectionpublic record CreateVectorBucketRequest(String vectorBucketName,- Object encryptionConfiguration+ Object encryptionConfiguration,+ Map<String, String> tags) {}@RegisterForReflection@@ -67,6 +70,7 @@ public class S3VectorsController {public record VectorBucketRepresentation(String vectorBucketName,String vectorBucketArn,+ double creationTime,Object encryptionConfiguration) {}@@ -98,10 +102,14 @@ public class S3VectorsController {String dataType,int dimension,String distanceMetric,- Object metadataConfiguration,- Object encryptionConfiguration+ MetadataConfiguration metadataConfiguration,+ Object encryptionConfiguration,+ Map<String, String> tags) {}+ @RegisterForReflection+ public record MetadataConfiguration(List<String> nonFilterableMetadataKeys) {}+@RegisterForReflectionpublic record CreateIndexResponse(String indexArn@@ -122,6 +130,7 @@ public class S3VectorsController {@RegisterForReflection@JsonInclude(JsonInclude.Include.NON_NULL)public record IndexRepresentation(+ double creationTime,String vectorBucketName,String indexName,String indexArn,@@ -256,7 +265,8 @@ public class S3VectorsController {@Path("/CreateVectorBucket")public Response createVectorBucket(@Context HttpHeaders headers, CreateVectorBucketRequest request) {String region = regionResolver.resolveRegion(headers);- VectorBucket bucket = service.createVectorBucket(request.vectorBucketName(), request.encryptionConfiguration(), region);+ VectorBucket bucket = service.createVectorBucket(request.vectorBucketName(), request.encryptionConfiguration(),+ request.tags() != null ? request.tags() : Map.of(), region);return Response.ok(new CreateVectorBucketResponse(bucket.getVectorBucketArn())).build();}@@ -265,7 +275,8 @@ public class S3VectorsController {public Response getVectorBucket(@Context HttpHeaders headers, GetVectorBucketRequest request) {String region = regionResolver.resolveRegion(headers);VectorBucket bucket = service.getVectorBucket(request.vectorBucketName(), region);- VectorBucketRepresentation rep = new VectorBucketRepresentation(bucket.getVectorBucketName(), bucket.getVectorBucketArn(), bucket.getEncryptionConfiguration());+ VectorBucketRepresentation rep = new VectorBucketRepresentation(bucket.getVectorBucketName(),+ bucket.getVectorBucketArn(), bucket.getCreationTime().toEpochMilli() / 1000.0, bucket.getEncryptionConfiguration());return Response.ok(new GetVectorBucketResponse(rep)).build();}@@ -275,7 +286,8 @@ public class S3VectorsController {String region = regionResolver.resolveRegion(headers);List<VectorBucket> buckets = service.listVectorBuckets(region);List<VectorBucketRepresentation> reps = buckets.stream()- .map(b -> new VectorBucketRepresentation(b.getVectorBucketName(), b.getVectorBucketArn(), b.getEncryptionConfiguration()))+ .map(b -> new VectorBucketRepresentation(b.getVectorBucketName(), b.getVectorBucketArn(),+ b.getCreationTime().toEpochMilli() / 1000.0, b.getEncryptionConfiguration())).toList();return Response.ok(new ListVectorBucketsResponse(reps, null)).build();}@@ -298,7 +310,10 @@ public class S3VectorsController {request.dimension(),request.dataType() != null ? request.dataType() : "float32",request.distanceMetric() != null ? request.distanceMetric() : "cosine",- List.of(),+ request.metadataConfiguration() != null && request.metadataConfiguration().nonFilterableMetadataKeys() != null+ ? request.metadataConfiguration().nonFilterableMetadataKeys() : List.of(),+ request.encryptionConfiguration(),+ request.tags() != null ? request.tags() : Map.of(),region);return Response.ok(new CreateIndexResponse(index.getIndexArn())).build();@@ -310,14 +325,16 @@ public class S3VectorsController {String region = regionResolver.resolveRegion(headers);VectorIndex index = service.getIndex(request.vectorBucketName(), request.indexName(), region);IndexRepresentation rep = new IndexRepresentation(+ index.getCreationTime().toEpochMilli() / 1000.0,index.getVectorBucketName(),index.getIndexName(),index.getIndexArn(),index.getDataType(),index.getDimension(),index.getDistanceMetric(),- null,- null+ index.getNonFilterableMetadataKeys().isEmpty()+ ? null : new MetadataConfiguration(index.getNonFilterableMetadataKeys()),+ index.getEncryptionConfiguration());return Response.ok(new GetIndexResponse(rep)).build();}@@ -329,6 +346,7 @@ public class S3VectorsController {List<VectorIndex> indexes = service.listIndexes(request.vectorBucketName(), region);List<IndexRepresentation> reps = indexes.stream().map(i -> new IndexRepresentation(+ i.getCreationTime().toEpochMilli() / 1000.0,i.getVectorBucketName(),i.getIndexName(),i.getIndexArn(),diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.javaindex 7d715bd34b..e73a4462c0 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java@@ -4,7 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;-import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.s3vectors.model.VectorBucket;import io.github.hectorvent.floci.services.s3vectors.model.VectorData;@@ -14,7 +14,15 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.nio.charset.StandardCharsets;-import java.util.*;+import java.time.Clock;+import java.time.Instant;+import java.util.ArrayList;+import java.util.Base64;+import java.util.Comparator;+import java.util.HashSet;+import java.util.List;+import java.util.Map;+import java.util.Optional;import java.util.concurrent.ConcurrentHashMap;import java.util.stream.Collectors;@@ -22,20 +30,36 @@ import java.util.stream.Collectors;public class S3VectorsService {private static final Logger LOG = Logger.getLogger(S3VectorsService.class);- private final StorageBackend<String, VectorBucket> store;+ private final AccountAwareStorageBackend<VectorBucket> store;private final RegionResolver regionResolver;+ private final Clock clock;+ // Bounded, stable monitors also serialize deletion and recreation of the same bucket.+ private final Object[] bucketLocks = new Object[256];@Injectpublic S3VectorsService(StorageFactory factory, RegionResolver regionResolver) {+ this(factory, regionResolver, Clock.systemUTC());+ }++ S3VectorsService(StorageFactory factory, RegionResolver regionResolver, Clock clock) {this.store = factory.create("s3vectors", "s3vectors.json",new TypeReference<Map<String, VectorBucket>>() {});this.regionResolver = regionResolver;+ this.clock = clock;+ for (int i = 0; i < bucketLocks.length; i++) {+ bucketLocks[i] = new Object();+ }}private String buildStorageKey(String region, String bucketName) {return region + "::" + bucketName;}+ private Object lockFor(String bucketName, String region) {+ String key = store.accountId() + "/" + buildStorageKey(region, bucketName);+ return bucketLocks[Math.floorMod(key.hashCode(), bucketLocks.length)];+ }+private String buildBucketArn(String region, String bucketName) {return AwsArnUtils.Arn.of("s3vectors", region, regionResolver.getAccountId(), "bucket/" + bucketName).toString();}@@ -45,68 +69,144 @@ public class S3VectorsService {}public VectorBucket createVectorBucket(String bucketName, Object encryptionConfiguration, String region) {- String key = buildStorageKey(region, bucketName);- if (store.get(key).isPresent()) {- throw new AwsException("ConflictException", "The vector bucket " + bucketName + " already exists.", 409);+ return createVectorBucket(bucketName, encryptionConfiguration, Map.of(), region);+ }++ public VectorBucket createVectorBucket(String bucketName, Object encryptionConfiguration,+ Map<String, String> tags, String region) {+ synchronized (lockFor(bucketName, region)) {+ String key = buildStorageKey(region, bucketName);+ if (store.get(key).isPresent()) {+ throw new AwsException("ConflictException", "The vector bucket " + bucketName + " already exists.", 409);+ }+ String arn = buildBucketArn(region, bucketName);+ VectorBucket bucket = new VectorBucket(bucketName, arn,+ encryptionConfiguration != null ? encryptionConfiguration : Map.of("sseType", "AES256"));+ bucket.setCreationTime(clock.instant());+ bucket.setTags(new ConcurrentHashMap<>(tags));+ store.put(key, bucket);+ LOG.infov("Created Vector Bucket: {0} in region {1}", bucketName, region);+ return bucket;}- String arn = buildBucketArn(region, bucketName);- VectorBucket bucket = new VectorBucket(bucketName, arn, encryptionConfiguration);- store.put(key, bucket);- LOG.infov("Created Vector Bucket: {0} in region {1}", bucketName, region);- return bucket;}public VectorBucket getVectorBucket(String bucketName, String region) {- String key = buildStorageKey(region, bucketName);- return store.get(key).orElseThrow(() ->+ return findVectorBucket(bucketName, region).orElseThrow(() ->new AwsException("NotFoundException", "The vector bucket " + bucketName + " does not exist.", 404));}public List<VectorBucket> listVectorBuckets(String region) {- return store.scan(k -> k.startsWith(region + "::")).stream()+ String prefix = region + "::";+ return store.keys().stream().filter(key -> key.startsWith(prefix))+ .flatMap(key -> findVectorBucket(key.substring(prefix.length()), region).stream()).sorted(Comparator.comparing(VectorBucket::getVectorBucketName)).collect(Collectors.toList());}+ private Optional<VectorBucket> findVectorBucket(String bucketName, String region) {+ synchronized (lockFor(bucketName, region)) {+ String key = buildStorageKey(region, bucketName);+ return store.get(key).map(bucket -> {+ if (backfillLegacyMetadata(bucket)) {+ store.put(key, bucket);+ }+ return bucket;+ });+ }+ }++ private boolean backfillLegacyMetadata(VectorBucket bucket) {+ boolean changed = false;+ Instant migrationTime = clock.instant();+ if (bucket.getCreationTime() == null) {+ bucket.setCreationTime(migrationTime);+ changed = true;+ }+ if (bucket.getEncryptionConfiguration() == null) {+ bucket.setEncryptionConfiguration(Map.of("sseType", "AES256"));+ changed = true;+ }+ for (VectorIndex index : bucket.getIndexes().values()) {+ if (index.getCreationTime() == null) {+ index.setCreationTime(migrationTime);+ changed = true;+ }+ if (index.getEncryptionConfiguration() == null) {+ index.setEncryptionConfiguration(bucket.getEncryptionConfiguration());+ changed = true;+ }+ }+ return changed;+ }+public void deleteVectorBucket(String bucketName, String region) {- String key = buildStorageKey(region, bucketName);- VectorBucket bucket = store.get(key).orElseThrow(() ->- new AwsException("NotFoundException", "The vector bucket " + bucketName + " does not exist.", 404));+ synchronized (lockFor(bucketName, region)) {+ String key = buildStorageKey(region, bucketName);+ VectorBucket bucket = store.get(key).orElseThrow(() ->+ new AwsException("NotFoundException", "The vector bucket " + bucketName + " does not exist.", 404));- if (!bucket.getIndexes().isEmpty()) {- throw new AwsException("ConflictException", "The vector bucket " + bucketName + " is not empty.", 409);+ if (!bucket.getIndexes().isEmpty()) {+ throw new AwsException("ConflictException", "The vector bucket " + bucketName + " is not empty.", 409);+ }+ store.delete(key);+ LOG.infov("Deleted Vector Bucket: {0}", bucketName);}- store.delete(key);- LOG.infov("Deleted Vector Bucket: {0}", bucketName);}public VectorIndex createIndex(String bucketName, String indexName, int dimension, String dataType,String distanceMetric, List<String> nonFilterableMetadataKeys, String region) {+ return createIndex(bucketName, indexName, dimension, dataType, distanceMetric,+ nonFilterableMetadataKeys, null, Map.of(), region);+ }++ public VectorIndex createIndex(String bucketName, String indexName, int dimension, String dataType,+ String distanceMetric, List<String> nonFilterableMetadataKeys,+ Object encryptionConfiguration, Map<String, String> tags, String region) {+ if (dimension < 1 || dimension > 4096) {+ throw new AwsException("ValidationException", "Dimension must be between 1 and 4096.", 400);+ }if (!"float32".equals(dataType)) {throw new AwsException("ValidationException", "Unsupported dataType: " + dataType + ". Only float32 is supported.", 400);}if (!"cosine".equals(distanceMetric) && !"euclidean".equals(distanceMetric)) {throw new AwsException("ValidationException", "Unsupported distanceMetric: " + distanceMetric + ". Supported metrics are cosine and euclidean.", 400);}-- VectorBucket bucket = getVectorBucket(bucketName, region);- if (bucket.getIndexes().containsKey(indexName)) {- throw new AwsException("ConflictException", "The index " + indexName + " already exists.", 409);+ if (nonFilterableMetadataKeys == null || nonFilterableMetadataKeys.size() > 10+ || nonFilterableMetadataKeys.stream().anyMatch(key -> key == null || key.isBlank() || key.length() > 63)+ || new HashSet<>(nonFilterableMetadataKeys).size() != nonFilterableMetadataKeys.size()) {+ throw new AwsException("ValidationException",+ "NonFilterableMetadataKeys must contain at most 10 unique keys of 1 to 63 characters.", 400);}- String arn = buildIndexArn(region, bucketName, indexName);- VectorIndex index = new VectorIndex(indexName, arn, bucketName, dimension, dataType, distanceMetric);- bucket.getIndexes().put(indexName, index);- store.put(buildStorageKey(region, bucketName), bucket);- LOG.infov("Created Index: {0} in bucket {1}", indexName, bucketName);- return index;+ synchronized (lockFor(bucketName, region)) {+ VectorBucket bucket = getVectorBucket(bucketName, region);+ if (bucket.getIndexes().containsKey(indexName)) {+ throw new AwsException("ConflictException", "The index " + indexName + " already exists.", 409);+ }++ String arn = buildIndexArn(region, bucketName, indexName);+ VectorIndex index = new VectorIndex(indexName, arn, bucketName, dimension, dataType, distanceMetric);+ index.setCreationTime(clock.instant());+ index.setNonFilterableMetadataKeys(List.copyOf(nonFilterableMetadataKeys));+ index.setEncryptionConfiguration(encryptionConfiguration != null+ ? encryptionConfiguration : bucket.getEncryptionConfiguration());+ index.setTags(new ConcurrentHashMap<>(tags));+ bucket.getIndexes().put(indexName, index);+ store.put(buildStorageKey(region, bucketName), bucket);+ LOG.infov("Created Index: {0} in bucket {1}", indexName, bucketName);+ return index;+ }}public VectorIndex getIndex(String bucketName, String indexName, String region) {- VectorBucket bucket = getVectorBucket(bucketName, region);+ return getIndex(getVectorBucket(bucketName, region), indexName);+ }++ private VectorIndex getIndex(VectorBucket bucket, String indexName) {VectorIndex index = bucket.getIndexes().get(indexName);if (index == null) {- throw new AwsException("NotFoundException", "The index " + indexName + " does not exist in bucket " + bucketName + ".", 404);+ throw new AwsException("NotFoundException",+ "The index " + indexName + " does not exist in bucket " + bucket.getVectorBucketName() + ".", 404);}return index;}@@ -119,28 +219,32 @@ public class S3VectorsService {}public void deleteIndex(String bucketName, String indexName, String region) {- VectorBucket bucket = getVectorBucket(bucketName, region);- if (!bucket.getIndexes().containsKey(indexName)) {- throw new AwsException("NotFoundException", "The index " + indexName + " does not exist.", 404);+ synchronized (lockFor(bucketName, region)) {+ VectorBucket bucket = getVectorBucket(bucketName, region);+ if (!bucket.getIndexes().containsKey(indexName)) {+ throw new AwsException("NotFoundException", "The index " + indexName + " does not exist.", 404);+ }+ bucket.getIndexes().remove(indexName);+ store.put(buildStorageKey(region, bucketName), bucket);+ LOG.infov("Deleted Index: {0}", indexName);}- bucket.getIndexes().remove(indexName);- store.put(buildStorageKey(region, bucketName), bucket);- LOG.infov("Deleted Index: {0}", indexName);}public void putVectors(String bucketName, String indexName, List<VectorData> vectors, String region) {- VectorBucket bucket = getVectorBucket(bucketName, region);- VectorIndex index = getIndex(bucketName, indexName, region);-- for (VectorData v : vectors) {- if (v.getData() != null && v.getData().size() != index.getDimension()) {- throw new AwsException("ValidationException",- "Vector dimension " + v.getData().size() + " does not match index dimension " + index.getDimension(), 400);+ synchronized (lockFor(bucketName, region)) {+ VectorBucket bucket = getVectorBucket(bucketName, region);+ VectorIndex index = getIndex(bucket, indexName);++ for (VectorData v : vectors) {+ if (v.getData() != null && v.getData().size() != index.getDimension()) {+ throw new AwsException("ValidationException",+ "Vector dimension " + v.getData().size() + " does not match index dimension " + index.getDimension(), 400);+ }+ index.getVectors().put(v.getKey(), v);}- index.getVectors().put(v.getKey(), v);+ store.put(buildStorageKey(region, bucketName), bucket);+ LOG.infov("Put {0} vectors into index {1}", vectors.size(), indexName);}- store.put(buildStorageKey(region, bucketName), bucket);- LOG.infov("Put {0} vectors into index {1}", vectors.size(), indexName);}public List<VectorData> getVectors(String bucketName, String indexName, List<String> keys, String region) {@@ -254,13 +358,15 @@ public class S3VectorsService {public record ListVectorsResult(List<VectorData> vectors, String nextToken) {}public void deleteVectors(String bucketName, String indexName, List<String> keys, String region) {- VectorBucket bucket = getVectorBucket(bucketName, region);- VectorIndex index = getIndex(bucketName, indexName, region);- for (String key : keys) {- index.getVectors().remove(key);+ synchronized (lockFor(bucketName, region)) {+ VectorBucket bucket = getVectorBucket(bucketName, region);+ VectorIndex index = getIndex(bucket, indexName);+ for (String key : keys) {+ index.getVectors().remove(key);+ }+ store.put(buildStorageKey(region, bucketName), bucket);+ LOG.infov("Deleted {0} vectors from index {1}", keys.size(), indexName);}- store.put(buildStorageKey(region, bucketName), bucket);- LOG.infov("Deleted {0} vectors from index {1}", keys.size(), indexName);}public List<QueryResult> queryVectors(String bucketName, String indexName, List<Float> queryVector,@@ -272,6 +378,7 @@ public class S3VectorsService {}S3VectorsMetadataFilter metadataFilter = S3VectorsMetadataFilter.compile(filter);+ validateFilterKeys(filter, index.getNonFilterableMetadataKeys());String metric = index.getDistanceMetric() != null ? index.getDistanceMetric().toLowerCase() : "cosine";List<QueryResult> results = new ArrayList<>();@@ -297,6 +404,23 @@ public class S3VectorsService {return results.stream().limit(topK).collect(Collectors.toList());}+ private static void validateFilterKeys(Object filter, List<String> nonFilterableKeys) {+ if (!(filter instanceof Map<?, ?> clauses)) {+ return;+ }+ for (Map.Entry<?, ?> clause : clauses.entrySet()) {+ if ("$and".equals(clause.getKey()) || "$or".equals(clause.getKey())) {+ if (clause.getValue() instanceof List<?> children) {+ for (Object child : children) {+ validateFilterKeys(child, nonFilterableKeys);+ }+ }+ } else if (nonFilterableKeys.contains(clause.getKey())) {+ throw new AwsException("ValidationException", "Cannot filter on non-filterable metadata key: " + clause.getKey(), 400);+ }+ }+ }+private double calculateCosineSimilarity(List<Float> v1, List<Float> v2) {if (v1 == null || v2 == null || v1.size() != v2.size()) return 0.0;double dotProduct = 0.0;diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorBucket.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorBucket.javaindex 6fd3e3a9f6..2111268daa 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorBucket.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorBucket.java@@ -2,11 +2,14 @@ package io.github.hectorvent.floci.services.s3vectors.model;import io.quarkus.runtime.annotations.RegisterForReflection;+import java.time.Instant;import java.util.Map;import java.util.concurrent.ConcurrentHashMap;@RegisterForReflectionpublic class VectorBucket {+ private Instant creationTime;+ private Map<String, String> tags = new ConcurrentHashMap<>();private String vectorBucketName;private String vectorBucketArn;private Object encryptionConfiguration;@@ -20,6 +23,12 @@ public class VectorBucket {this.encryptionConfiguration = encryptionConfiguration;}+ public Instant getCreationTime() { return creationTime; }+ public void setCreationTime(Instant creationTime) { this.creationTime = creationTime; }++ public Map<String, String> getTags() { return tags; }+ public void setTags(Map<String, String> tags) { this.tags = tags; }+public String getVectorBucketName() { return vectorBucketName; }public void setVectorBucketName(String vectorBucketName) { this.vectorBucketName = vectorBucketName; }diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorIndex.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorIndex.javaindex 2742d0f0af..69aa868060 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorIndex.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorIndex.java@@ -2,14 +2,20 @@ package io.github.hectorvent.floci.services.s3vectors.model;import io.quarkus.runtime.annotations.RegisterForReflection;+import java.time.Instant;+import java.util.List;import java.util.Map;import java.util.concurrent.ConcurrentHashMap;@RegisterForReflectionpublic class VectorIndex {+ private Instant creationTime;+ private Map<String, String> tags = new ConcurrentHashMap<>();private String indexName;private String indexArn;private String vectorBucketName;+ private Object encryptionConfiguration;+ private List<String> nonFilterableMetadataKeys = List.of();private int dimension;private String dataType;private String distanceMetric;@@ -26,6 +32,12 @@ public class VectorIndex {this.distanceMetric = distanceMetric;}+ public Instant getCreationTime() { return creationTime; }+ public void setCreationTime(Instant creationTime) { this.creationTime = creationTime; }++ public Map<String, String> getTags() { return tags; }+ public void setTags(Map<String, String> tags) { this.tags = tags; }+public String getIndexName() { return indexName; }public void setIndexName(String indexName) { this.indexName = indexName; }@@ -35,6 +47,12 @@ public class VectorIndex {public String getVectorBucketName() { return vectorBucketName; }public void setVectorBucketName(String vectorBucketName) { this.vectorBucketName = vectorBucketName; }+ public Object getEncryptionConfiguration() { return encryptionConfiguration; }+ public void setEncryptionConfiguration(Object encryptionConfiguration) { this.encryptionConfiguration = encryptionConfiguration; }++ public List<String> getNonFilterableMetadataKeys() { return nonFilterableMetadataKeys; }+ public void setNonFilterableMetadataKeys(List<String> keys) { this.nonFilterableMetadataKeys = keys; }+public int getDimension() { return dimension; }public void setDimension(int dimension) { this.dimension = dimension; }diff --git a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsLegacySdkIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsLegacySdkIntegrationTest.javanew file mode 100644index 0000000000..782acff4cb--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsLegacySdkIntegrationTest.java@@ -0,0 +1,86 @@+package io.github.hectorvent.floci.services.s3vectors;++import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.s3vectors.model.VectorBucket;+import io.github.hectorvent.floci.services.s3vectors.model.VectorIndex;+import io.quarkus.test.common.http.TestHTTPResource;+import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;+import software.amazon.awssdk.regions.Region;+import software.amazon.awssdk.services.s3vectors.S3VectorsClient;+import software.amazon.awssdk.services.s3vectors.model.GetVectorBucketResponse;+import software.amazon.awssdk.services.s3vectors.model.Index;++import java.net.URI;+import java.util.List;+import java.util.Map;+import java.util.UUID;++import static org.junit.jupiter.api.Assertions.*;++@QuarkusTest+class S3VectorsLegacySdkIntegrationTest {+ @TestHTTPResource+ URI endpoint;++ @Inject+ S3VectorsService service;++ @Inject+ StorageFactory factory;++ @Inject+ ObjectMapper mapper;++ @Test+ void legacyBucketAndIndexMetadataIsAvailableThroughSdkGetAndList() throws Exception {+ String region = "us-east-1";+ String name = "legacy-vectors-" + UUID.randomUUID().toString().substring(0, 8);+ service.createVectorBucket(name, null, region);+ service.createIndex(name, "old", 1, "float32", "cosine", List.of(), region);+ ObjectNode legacyJson = mapper.valueToTree(service.getVectorBucket(name, region));+ legacyJson.remove(List.of("creationTime", "encryptionConfiguration"));+ ObjectNode legacyIndex = (ObjectNode) legacyJson.path("indexes").path("old");+ legacyIndex.remove(List.of("creationTime", "encryptionConfiguration"));+ AccountAwareStorageBackend<VectorBucket> store = factory.create("s3vectors", "s3vectors.json",+ new TypeReference<Map<String, VectorBucket>>() {});+ store.put(region + "::" + name, mapper.treeToValue(legacyJson, VectorBucket.class));++ try (S3VectorsClient vectors = S3VectorsClient.builder().endpointOverride(endpoint).region(Region.of(region))+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("test", "test")))+ .httpClientBuilder(UrlConnectionHttpClient.builder()).build()) {+ try {+ // Creating an index must also resolve the legacy bucket's missing encryption default.+ vectors.createIndex(request -> request.vectorBucketName(name).indexName("new")+ .dimension(1).dataType("float32").distanceMetric("cosine"));+ GetVectorBucketResponse bucket = vectors.getVectorBucket(request -> request.vectorBucketName(name));+ assertEquals("AES256", bucket.vectorBucket().encryptionConfiguration().sseTypeAsString());+ assertNotNull(bucket.vectorBucket().creationTime());+ assertEquals(bucket.vectorBucket().creationTime(), vectors.listVectorBuckets(request -> request.maxResults(100))+ .vectorBuckets().stream().filter(item -> name.equals(item.vectorBucketName()))+ .findFirst().orElseThrow().creationTime());+ for (String indexName : List.of("old", "new")) {+ Index index = vectors.getIndex(request -> request.vectorBucketName(name).indexName(indexName)).index();+ assertEquals("AES256", index.encryptionConfiguration().sseTypeAsString());+ assertNotNull(index.creationTime());+ assertEquals(index.creationTime(), vectors.listIndexes(request -> request.vectorBucketName(name))+ .indexes().stream().filter(item -> indexName.equals(item.indexName()))+ .findFirst().orElseThrow().creationTime());+ }+ } finally {+ for (VectorIndex index : service.listIndexes(name, region)) {+ vectors.deleteIndex(request -> request.vectorBucketName(name).indexName(index.getIndexName()));+ }+ vectors.deleteVectorBucket(request -> request.vectorBucketName(name));+ }+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsPersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsPersistenceTest.javanew file mode 100644index 0000000000..34d2e84fb6--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsPersistenceTest.java@@ -0,0 +1,133 @@+package io.github.hectorvent.floci.services.s3vectors;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.HybridStorage;+import io.github.hectorvent.floci.core.storage.PersistentStorage;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.core.storage.WalStorage;+import io.github.hectorvent.floci.services.s3vectors.model.VectorBucket;+import io.github.hectorvent.floci.services.s3vectors.model.VectorIndex;+import org.junit.jupiter.api.io.TempDir;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;++import java.nio.file.Files;+import java.nio.file.Path;+import java.time.Clock;+import java.time.Instant;+import java.time.ZoneOffset;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.Mockito.*;++class S3VectorsPersistenceTest {+ private static final String REGION = "us-east-1";+ private static final String ACCOUNT = "000000000000";+ private static final String KEY = ACCOUNT + "/" + REGION + "::legacy";+ private static final Map<String, String> DEFAULT_ENCRYPTION = Map.of("sseType", "AES256");+ private static final Instant KNOWN_TIME = Instant.parse("2026-01-01T00:00:00Z");+ private static final Instant MIGRATION_TIME = Instant.parse("2026-02-01T00:00:00Z");++ @ParameterizedTest+ @CsvSource({"persistent,false", "persistent,true", "hybrid,false", "hybrid,true", "wal,false", "wal,true"})+ void legacyMetadataIsSavedOnceAndSurvivesReload(String mode, boolean listFirst, @TempDir Path directory)+ throws Exception {+ Path snapshot = directory.resolve("s3vectors.json");+ Files.writeString(snapshot, """+ {+ "000000000000/us-east-1::legacy": {+ "vectorBucketName": "legacy",+ "encryptionConfiguration": null,+ "indexes": {+ "old": {"indexName": "old", "dimension": 1, "dataType": "float32", "distanceMetric": "cosine"},+ "configured": {+ "indexName": "configured",+ "creationTime": "2026-01-01T00:00:00Z",+ "encryptionConfiguration": {"sseType": "aws:kms", "kmsKeyArn": "key"}+ }+ }+ }+ }+ """);+ StorageBackend<String, VectorBucket> first = open(mode, snapshot);+ try {+ VectorBucket legacy = first.get(KEY).orElseThrow();+ assertNull(legacy.getCreationTime());+ assertNull(legacy.getIndexes().get("old").getCreationTime());+ S3VectorsService service = service(first, Clock.fixed(MIGRATION_TIME, ZoneOffset.UTC));+ VectorBucket bucket = listFirst ? service.listVectorBuckets(REGION).getFirst()+ : service.getVectorBucket("legacy", REGION);+ VectorIndex index = service.getIndex("legacy", "old", REGION);+ assertEquals(MIGRATION_TIME, bucket.getCreationTime());+ assertEquals(MIGRATION_TIME, index.getCreationTime());+ assertEquals(DEFAULT_ENCRYPTION, bucket.getEncryptionConfiguration());+ assertEquals(DEFAULT_ENCRYPTION, index.getEncryptionConfiguration());+ assertEquals(KNOWN_TIME, bucket.getIndexes().get("configured").getCreationTime());+ assertEquals(Map.of("sseType", "aws:kms", "kmsKeyArn", "key"),+ bucket.getIndexes().get("configured").getEncryptionConfiguration());+ assertEquals(MIGRATION_TIME, service.listVectorBuckets(REGION).getFirst().getCreationTime());+ assertEquals(MIGRATION_TIME, service.listIndexes("legacy", REGION).stream()+ .filter(item -> "old".equals(item.getIndexName())).findFirst().orElseThrow().getCreationTime());+ verify(first, times(1)).put(eq(KEY), any());++ VectorIndex created = service.createIndex("legacy", "new", 1, "float32", "cosine", List.of(), REGION);+ assertEquals(DEFAULT_ENCRYPTION, created.getEncryptionConfiguration());+ assertEquals(MIGRATION_TIME, created.getCreationTime());+ } finally {+ close(first);+ }++ StorageBackend<String, VectorBucket> second = open(mode, snapshot);+ try {+ VectorBucket stored = second.get(KEY).orElseThrow();+ assertEquals(MIGRATION_TIME, stored.getCreationTime());+ assertEquals(MIGRATION_TIME, stored.getIndexes().get("old").getCreationTime());+ assertEquals(DEFAULT_ENCRYPTION, stored.getEncryptionConfiguration());+ S3VectorsService restarted = service(second, Clock.fixed(MIGRATION_TIME.plusSeconds(86_400), ZoneOffset.UTC));+ assertEquals(MIGRATION_TIME, restarted.getVectorBucket("legacy", REGION).getCreationTime());+ assertEquals(MIGRATION_TIME, restarted.listVectorBuckets(REGION).getFirst().getCreationTime());+ assertEquals(MIGRATION_TIME, restarted.getIndex("legacy", "old", REGION).getCreationTime());+ assertEquals(MIGRATION_TIME, restarted.getIndex("legacy", "new", REGION).getCreationTime());+ assertEquals(KNOWN_TIME, restarted.getIndex("legacy", "configured", REGION).getCreationTime());+ assertEquals(DEFAULT_ENCRYPTION, restarted.getIndex("legacy", "new", REGION).getEncryptionConfiguration());+ assertEquals(3, restarted.listIndexes("legacy", REGION).size());+ verify(second, never()).put(anyString(), any());+ } finally {+ close(second);+ }+ }++ private static S3VectorsService service(StorageBackend<String, VectorBucket> backend, Clock clock) {+ StorageFactory factory = mock(StorageFactory.class);+ when(factory.<VectorBucket>create(anyString(), anyString(), any()))+ .thenReturn(new AccountAwareStorageBackend<>(backend, null, ACCOUNT));+ return new S3VectorsService(factory, new RegionResolver(REGION, ACCOUNT), clock);+ }++ private static StorageBackend<String, VectorBucket> open(String mode, Path snapshot) {+ TypeReference<Map<String, VectorBucket>> type = new TypeReference<>() {};+ StorageBackend<String, VectorBucket> backend = spy(switch (mode) {+ case "persistent" -> new PersistentStorage<>(snapshot, type);+ case "hybrid" -> new HybridStorage<>(snapshot, type, 60_000);+ case "wal" -> new WalStorage<>(snapshot, snapshot.resolveSibling("s3vectors.wal"), type, 60_000);+ default -> throw new IllegalArgumentException(mode);+ });+ backend.load();+ return backend;+ }++ private static void close(StorageBackend<String, VectorBucket> backend) {+ if (backend instanceof HybridStorage<?, ?> hybrid) {+ hybrid.shutdown();+ } else if (backend instanceof WalStorage<?, ?> wal) {+ wal.shutdown();+ } else {+ backend.flush();+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsSdkIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsSdkIntegrationTest.javanew file mode 100644index 0000000000..4cd8c7926a--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsSdkIntegrationTest.java@@ -0,0 +1,236 @@+package io.github.hectorvent.floci.services.s3vectors;++import io.github.hectorvent.floci.services.s3vectors.model.VectorIndex;+import io.quarkus.test.common.http.TestHTTPResource;+import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.Arguments;+import org.junit.jupiter.params.provider.MethodSource;+import org.junit.jupiter.params.provider.ValueSource;+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;+import software.amazon.awssdk.core.document.Document;+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;+import software.amazon.awssdk.regions.Region;+import software.amazon.awssdk.services.s3vectors.S3VectorsClient;+import software.amazon.awssdk.services.s3vectors.model.Index;+import software.amazon.awssdk.services.s3vectors.model.PutInputVector;+import software.amazon.awssdk.services.s3vectors.model.QueryVectorsResponse;+import software.amazon.awssdk.services.s3vectors.model.ValidationException;+import software.amazon.awssdk.services.s3vectors.model.VectorBucket;+import software.amazon.awssdk.services.s3vectors.model.VectorData;++import java.net.URI;+import java.util.Collections;+import java.util.List;+import java.util.Map;+import java.util.UUID;+import java.util.stream.Stream;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.*;++@QuarkusTest+class S3VectorsSdkIntegrationTest {++ @TestHTTPResource+ URI endpoint;++ @Inject+ S3VectorsService service;++ @Test+ void sdkCreatedResourcesPreserveConfigurationAndRejectNonFilterableQueries() {+ String name = "sdk-vectors-" + UUID.randomUUID().toString().substring(0, 8);+ try (S3VectorsClient vectors = vectors("us-east-1")) {+ vectors.createVectorBucket(request -> request.vectorBucketName(name)+ .encryptionConfiguration(encryption -> encryption.sseType("AES256"))+ .tags(Map.of("team", "search")));+ try {+ vectors.createIndex(request -> request.vectorBucketName(name).indexName("index")+ .dataType("float32").dimension(3).distanceMetric("cosine")+ .metadataConfiguration(metadata -> metadata.nonFilterableMetadataKeys("text"))+ .tags(Map.of("team", "search")));+ Index index = vectors.getIndex(request -> request.vectorBucketName(name).indexName("index")).index();+ VectorBucket bucket = vectors.getVectorBucket(request -> request.vectorBucketName(name)).vectorBucket();+ assertNotNull(bucket.creationTime());+ assertNotNull(index.creationTime());+ assertEquals(service.getVectorBucket(name, "us-east-1").getCreationTime().toEpochMilli(),+ bucket.creationTime().toEpochMilli());+ assertEquals(service.getIndex(name, "index", "us-east-1").getCreationTime().toEpochMilli(),+ index.creationTime().toEpochMilli());+ assertEquals(bucket.creationTime(), vectors.listVectorBuckets(request -> request.maxResults(100))+ .vectorBuckets().stream()+ .filter(item -> name.equals(item.vectorBucketName())).findFirst().orElseThrow().creationTime());+ assertEquals(index.creationTime(), vectors.listIndexes(request -> request.vectorBucketName(name))+ .indexes().getFirst().creationTime());+ assertEquals(List.of("text"), index.metadataConfiguration().nonFilterableMetadataKeys());+ assertEquals("AES256", index.encryptionConfiguration().sseTypeAsString());+ assertEquals(Map.of("team", "search"), service.getVectorBucket(name, "us-east-1").getTags());+ assertEquals(Map.of("team", "search"), service.getIndex(name, "index", "us-east-1").getTags());+ assertThrows(ValidationException.class, () -> vectors.queryVectors(request -> request+ .vectorBucketName(name).indexName("index").topK(1)+ .queryVector(VectorData.builder().float32(1.0f, 0.0f, 0.0f).build())+ .filter(Document.fromMap(Map.of("text", Document.fromString("hello"))))));+ } finally {+ for (String index : service.listIndexes(name, "us-east-1").stream()+ .map(VectorIndex::getIndexName).toList()) {+ vectors.deleteIndex(request -> request.vectorBucketName(name).indexName(index));+ }+ vectors.deleteVectorBucket(request -> request.vectorBucketName(name));+ }+ }+ }++ static Stream<Arguments> invalidIndexSettings() {+ return Stream.of(+ Arguments.of("float64", "cosine", List.of(), "dataType"),+ Arguments.of("float32", "manhattan", List.of(), "distanceMetric"),+ Arguments.of("float32", "cosine", List.of(""), "NonFilterableMetadataKeys"),+ Arguments.of("float32", "cosine", List.of(" "), "NonFilterableMetadataKeys"),+ Arguments.of("float32", "cosine", Collections.singletonList(null), "NonFilterableMetadataKeys"),+ Arguments.of("float32", "cosine", List.of("a".repeat(64)), "NonFilterableMetadataKeys"),+ Arguments.of("float32", "cosine", List.of("text", "text"), "NonFilterableMetadataKeys"),+ Arguments.of("float32", "cosine", Collections.nCopies(11, "key"), "NonFilterableMetadataKeys"));+ }++ @ParameterizedTest+ @ValueSource(strings = {"$and", "$or"})+ void logicalOperatorsAreNotTreatedAsNonFilterableFields(String operator) {+ String name = "logical-vectors-" + UUID.randomUUID().toString().substring(0, 8);+ try (S3VectorsClient vectors = vectors("us-east-1")) {+ vectors.createVectorBucket(request -> request.vectorBucketName(name));+ try {+ vectors.createIndex(request -> request.vectorBucketName(name).indexName("index")+ .dataType("float32").dimension(1).distanceMetric("cosine")+ .metadataConfiguration(metadata -> metadata.nonFilterableMetadataKeys("$and", "$or", "text")));+ vectors.putVectors(request -> request.vectorBucketName(name).indexName("index")+ .vectors(PutInputVector.builder().key("match")+ .data(VectorData.builder().float32(1.0f).build())+ .metadata(Document.fromMap(Map.of("category", Document.fromString("x")))).build()));+ Document filter = Document.fromMap(Map.of(operator, Document.fromList(List.of(+ Document.fromMap(Map.of("category", Document.fromString("x")))))));+ QueryVectorsResponse result = vectors.queryVectors(request -> request.vectorBucketName(name)+ .indexName("index").queryVector(VectorData.builder().float32(1.0f).build()).topK(1).filter(filter));+ assertEquals(List.of("match"), result.vectors().stream().map(item -> item.key()).toList());++ Document forbidden = Document.fromMap(Map.of(operator, Document.fromList(List.of(+ Document.fromMap(Map.of("$and", Document.fromList(List.of(+ Document.fromMap(Map.of("text", Document.fromString("hidden")))))))))));+ ValidationException error = assertThrows(ValidationException.class, () -> vectors.queryVectors(request -> request+ .vectorBucketName(name).indexName("index").queryVector(VectorData.builder().float32(1.0f).build())+ .topK(1).filter(forbidden)));+ assertEquals(400, error.statusCode());+ assertTrue(error.getMessage().contains("text"));+ } finally {+ for (VectorIndex index : service.listIndexes(name, "us-east-1")) {+ vectors.deleteIndex(request -> request.vectorBucketName(name).indexName(index.getIndexName()));+ }+ vectors.deleteVectorBucket(request -> request.vectorBucketName(name));+ }+ }+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void configurationAppearsOnlyInGetIndexAndEmptyMetadataIsOmitted(boolean configured) {+ String name = "response-vectors-" + UUID.randomUUID().toString().substring(0, 8);+ try (S3VectorsClient vectors = vectors("us-east-1")) {+ vectors.createVectorBucket(request -> request.vectorBucketName(name));+ try {+ vectors.createIndex(request -> {+ request.vectorBucketName(name).indexName("index").dimension(3)+ .dataType("float32").distanceMetric("cosine");+ if (configured) {+ request.metadataConfiguration(metadata -> metadata.nonFilterableMetadataKeys("text"));+ }+ });+ Index index = vectors.getIndex(request -> request.vectorBucketName(name).indexName("index")).index();+ assertEquals("AES256", index.encryptionConfiguration().sseTypeAsString());+ if (configured) {+ assertEquals(List.of("text"), index.metadataConfiguration().nonFilterableMetadataKeys());+ } else {+ assertNull(index.metadataConfiguration());+ }++ Map<String, Object> detail = given().baseUri(endpoint.toString()).contentType("application/json")+ .body(Map.of("vectorBucketName", name, "indexName", "index"))+ .post("/GetIndex").then().statusCode(200).extract().path("index");+ assertEquals(configured, detail.containsKey("metadataConfiguration"));+ assertTrue(detail.containsKey("encryptionConfiguration"));+ // SDK clients ignore unknown fields, so check the wire response as well.+ Map<String, Object> summary = given().baseUri(endpoint.toString()).contentType("application/json")+ .body(Map.of("vectorBucketName", name))+ .post("/ListIndexes").then().statusCode(200).extract().path("indexes[0]");+ assertEquals(index.indexArn(), summary.get("indexArn"));+ assertFalse(summary.containsKey("metadataConfiguration"));+ assertFalse(summary.containsKey("encryptionConfiguration"));+ } finally {+ for (VectorIndex index : service.listIndexes(name, "us-east-1")) {+ vectors.deleteIndex(request -> request.vectorBucketName(name).indexName(index.getIndexName()));+ }+ vectors.deleteVectorBucket(request -> request.vectorBucketName(name));+ }+ }+ }++ @ParameterizedTest+ @MethodSource("invalidIndexSettings")+ void invalidIndexSettingsReturnValidationErrors(String dataType, String distanceMetric,+ List<String> keys, String errorMember) {+ String name = "invalid-vectors-" + UUID.randomUUID().toString().substring(0, 8);+ try (S3VectorsClient vectors = vectors("us-east-1")) {+ vectors.createVectorBucket(request -> request.vectorBucketName(name));+ try {+ ValidationException error = assertThrows(ValidationException.class,+ () -> vectors.createIndex(request -> request.vectorBucketName(name).indexName("index")+ .dimension(3).dataType(dataType).distanceMetric(distanceMetric)+ .metadataConfiguration(metadata -> metadata.nonFilterableMetadataKeys(keys))));+ assertEquals(400, error.statusCode());+ assertTrue(error.getMessage().contains(errorMember), error.getMessage());+ assertTrue(vectors.listIndexes(request -> request.vectorBucketName(name)).indexes().isEmpty());+ } finally {+ for (VectorIndex index : service.listIndexes(name, "us-east-1")) {+ vectors.deleteIndex(request -> request.vectorBucketName(name).indexName(index.getIndexName()));+ }+ vectors.deleteVectorBucket(request -> request.vectorBucketName(name));+ }+ }+ }++ @ParameterizedTest+ @ValueSource(ints = {0, 1, 4096, 4097})+ void dimensionBoundariesAreEnforcedThroughTheSdk(int dimension) {+ String name = "dimension-vectors-" + UUID.randomUUID().toString().substring(0, 8);+ try (S3VectorsClient vectors = vectors("us-east-1")) {+ vectors.createVectorBucket(request -> request.vectorBucketName(name));+ try {+ if (dimension == 0 || dimension == 4097) {+ ValidationException error = assertThrows(ValidationException.class,+ () -> vectors.createIndex(request -> request.vectorBucketName(name).indexName("index")+ .dimension(dimension).dataType("float32").distanceMetric("cosine")));+ assertEquals(400, error.statusCode());+ assertTrue(vectors.listIndexes(request -> request.vectorBucketName(name)).indexes().isEmpty());+ } else {+ vectors.createIndex(request -> request.vectorBucketName(name).indexName("index")+ .dimension(dimension).dataType("float32").distanceMetric("cosine"));+ assertEquals(dimension, vectors.getIndex(request -> request.vectorBucketName(name)+ .indexName("index")).index().dimension());+ }+ } finally {+ for (VectorIndex index : service.listIndexes(name, "us-east-1")) {+ vectors.deleteIndex(request -> request.vectorBucketName(name).indexName(index.getIndexName()));+ }+ vectors.deleteVectorBucket(request -> request.vectorBucketName(name));+ }+ }+ }++ private S3VectorsClient vectors(String region) {+ return S3VectorsClient.builder().endpointOverride(endpoint).region(Region.of(region))+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("test", "test")))+ .httpClientBuilder(UrlConnectionHttpClient.builder()).build();+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsServiceTest.javanew file mode 100644index 0000000000..660af4a2ea--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsServiceTest.java@@ -0,0 +1,114 @@+package io.github.hectorvent.floci.services.s3vectors;++import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.s3vectors.model.VectorBucket;+import io.github.hectorvent.floci.services.s3vectors.model.VectorIndex;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.time.Clock;+import java.time.Instant;+import java.time.ZoneOffset;+import java.util.Collections;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.*;++class S3VectorsServiceTest {++ private static final String REGION = "us-east-1";+ private static final Instant CREATION_TIME = Instant.parse("2026-02-01T00:00:00Z");+ private S3VectorsService service;++ @BeforeEach+ void setUp() {+ StorageFactory factory = new StorageFactory(null, null) {+ @Override+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return AccountAwareStorageBackend.inMemory("000000000000");+ }+ };+ service = new S3VectorsService(factory, new RegionResolver(REGION, "000000000000"),+ Clock.fixed(CREATION_TIME, ZoneOffset.UTC));+ }++ @Test+ void persistsCreationTimesConfigurationAndCreationTags() throws Exception {+ Map<String, String> encryption = Map.of("sseType", "aws:kms", "kmsKeyArn", "key");+ VectorBucket bucket = service.createVectorBucket("bucket", encryption, Map.of("old", "value"), REGION);+ VectorIndex index = service.createIndex("bucket", "index", 3, "float32", "cosine", List.of("text"),+ null, Map.of("old", "value"), REGION);+ assertEquals(CREATION_TIME, bucket.getCreationTime());+ assertEquals(CREATION_TIME, index.getCreationTime());++ ObjectMapper mapper = new ObjectMapper().findAndRegisterModules();+ VectorBucket restored = mapper.readValue(mapper.writeValueAsString(bucket), VectorBucket.class);+ VectorIndex restoredIndex = restored.getIndexes().get("index");+ assertEquals(bucket.getCreationTime(), restored.getCreationTime());+ assertEquals(index.getCreationTime(), restoredIndex.getCreationTime());+ assertEquals(encryption, restoredIndex.getEncryptionConfiguration());+ assertEquals(List.of("text"), restoredIndex.getNonFilterableMetadataKeys());+ assertEquals(Map.of("old", "value"), restored.getTags());+ assertEquals(Map.of("old", "value"), restoredIndex.getTags());+ }++ @Test+ void indexEncryptionCanOverrideTheBucketDefault() {+ VectorBucket bucket = service.createVectorBucket("bucket", null, REGION);+ assertEquals(Map.of("sseType", "AES256"), bucket.getEncryptionConfiguration());+ Map<String, String> encryption = Map.of("sseType", "aws:kms", "kmsKeyArn", "key");+ VectorIndex index = service.createIndex("bucket", "index", 3, "float32", "cosine", List.of(),+ encryption, Map.of(), REGION);+ assertEquals(encryption, index.getEncryptionConfiguration());+ }++ @Test+ void nonFilterableMetadataKeysAreRejectedEvenInsideLogicalFilters() {+ service.createVectorBucket("bucket", null, REGION);+ service.createIndex("bucket", "index", 3, "float32", "cosine", List.of("text"), REGION);+ for (Object filter : List.of(Map.of("text", "hello"), Map.of("$and", List.of(Map.of("text", "hello"))))) {+ AwsException error = assertThrows(AwsException.class,+ () -> service.queryVectors("bucket", "index", List.of(1.0f, 0.0f, 0.0f), filter, 1, REGION));+ assertEquals("ValidationException", error.getErrorCode());+ }+ assertEquals(List.of(), service.queryVectors("bucket", "index", List.of(1.0f, 0.0f, 0.0f),+ Map.of("category", "hello"), 1, REGION));+ }++ @Test+ void rejectsInvalidIndexDimensions() {+ service.createVectorBucket("bucket", null, REGION);+ for (int dimension : List.of(0, 4097)) {+ assertThrows(AwsException.class,+ () -> service.createIndex("bucket", "index", dimension, "float32", "cosine", List.of(), REGION));+ }+ assertTrue(service.listIndexes("bucket", REGION).isEmpty());+ }++ @Test+ void invalidMetadataKeysFailBeforeCreatingAnIndex() {+ service.createVectorBucket("bucket", null, REGION);+ for (List<String> keys : List.of(List.of(""), List.of(" "), List.of("a".repeat(64)),+ Collections.nCopies(11, "key"), List.of("text", "text"), Collections.<String>singletonList(null))) {+ AwsException error = assertThrows(AwsException.class,+ () -> service.createIndex("bucket", "index", 3, "float32", "cosine", keys, REGION));+ assertEquals("ValidationException", error.getErrorCode());+ }+ assertTrue(service.listIndexes("bucket", REGION).isEmpty());+ }++ @Test+ void metadataKeyLimitsAreInclusive() {+ service.createVectorBucket("bucket", null, REGION);+ List<String> keys = List.of("a".repeat(63), "b", "c", "d", "e", "f", "g", "h", "i", "j");+ VectorIndex index = service.createIndex("bucket", "index", 3, "float32", "euclidean", keys, REGION);+ assertEquals(keys, index.getNonFilterableMetadataKeys());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsWriteLockTest.java b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsWriteLockTest.javanew file mode 100644index 0000000000..e9f9bb7e90--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsWriteLockTest.java@@ -0,0 +1,179 @@+package io.github.hectorvent.floci.services.s3vectors;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.s3vectors.model.VectorBucket;+import io.github.hectorvent.floci.services.s3vectors.model.VectorData;+import io.github.hectorvent.floci.testing.ThreadStates;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.util.List;+import java.util.Map;+import java.util.Optional;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.FutureTask;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.ArgumentMatchers.*;+import static org.mockito.Mockito.*;++class S3VectorsWriteLockTest {++ private static final String REGION = "us-east-1";+ private ParkingStorage storage;+ private S3VectorsService service;++ private void setUp(boolean copyOnAccess) {+ storage = new ParkingStorage(copyOnAccess);+ StorageFactory factory = mock(StorageFactory.class);+ when(factory.<VectorBucket>create(anyString(), anyString(), any()))+ .thenReturn(new AccountAwareStorageBackend<>(storage, null, "000000000000"));+ service = new S3VectorsService(factory, new RegionResolver(REGION, "000000000000"));+ service.createVectorBucket("bucket", null, REGION);+ }++ private void createIndex() {+ service.createIndex("bucket", "index", 1, "float32", "cosine", List.of(), REGION);+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void vectorWritesPreserveConcurrentIndexCreation(boolean copyOnAccess) throws Exception {+ setUp(copyOnAccess);+ createIndex();+ overlap(() -> putVector("first"), () -> service.createIndex("bucket", "second", 1,+ "float32", "cosine", List.of(), REGION));+ assertEquals("second", service.getIndex("bucket", "second", REGION).getIndexName());+ assertEquals(1, service.getVectors("bucket", "index", List.of("first"), REGION).size());+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void concurrentVectorWritesPreserveBothVectors(boolean copyOnAccess) throws Exception {+ setUp(copyOnAccess);+ createIndex();+ overlap(() -> putVector("first"), () -> putVector("second"));+ assertEquals(2, service.getVectors("bucket", "index", List.of("first", "second"), REGION).size());+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void vectorWritesDoNotReviveDeletedVectorsOrIndexes(boolean copyOnAccess) throws Exception {+ setUp(copyOnAccess);+ createIndex();+ putVector("old");+ overlap(() -> putVector("new"), () -> service.deleteVectors("bucket", "index", List.of("old"), REGION));+ assertTrue(service.getVectors("bucket", "index", List.of("old"), REGION).isEmpty());+ assertEquals(1, service.getVectors("bucket", "index", List.of("new"), REGION).size());++ overlap(() -> putVector("new"), () -> service.deleteIndex("bucket", "index", REGION));+ AwsException error = assertThrows(AwsException.class, () -> service.getIndex("bucket", "index", REGION));+ assertEquals("NotFoundException", error.getErrorCode());+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void indexDeletionCannotResurrectADeletedOrRecreatedBucket(boolean copyOnAccess) throws Exception {+ setUp(copyOnAccess);+ createIndex();+ overlap(() -> service.deleteIndex("bucket", "index", REGION), () -> service.deleteVectorBucket("bucket", REGION));+ assertThrows(AwsException.class, () -> service.getVectorBucket("bucket", REGION));++ service.createVectorBucket("bucket", null, REGION);+ createIndex();+ overlap(() -> service.deleteIndex("bucket", "index", REGION), () -> {+ service.deleteVectorBucket("bucket", REGION);+ service.createVectorBucket("bucket", null, Map.of("new", "tag"), REGION);+ });+ assertEquals(Map.of("new", "tag"), service.getVectorBucket("bucket", REGION).getTags());+ }++ private void putVector(String key) {+ service.putVectors("bucket", "index", List.of(new VectorData(key, List.of(1.0f), Map.of())), REGION);+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void legacyMetadataMigrationDoesNotOverwriteConcurrentWrites(boolean copyOnAccess) throws Exception {+ setUp(copyOnAccess);+ VectorBucket legacy = new VectorBucket("bucket", "bucket-arn", null);+ storage.put("000000000000/" + REGION + "::bucket", legacy);+ overlap(() -> service.listVectorBuckets(REGION), this::createIndex);+ assertNotNull(service.getVectorBucket("bucket", REGION).getCreationTime());+ assertEquals(Map.of("sseType", "AES256"), service.getIndex("bucket", "index", REGION).getEncryptionConfiguration());+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void legacyMetadataMigrationDoesNotReviveADeletedBucket(boolean copyOnAccess) throws Exception {+ setUp(copyOnAccess);+ storage.put("000000000000/" + REGION + "::bucket", new VectorBucket("bucket", "bucket-arn", null));+ overlap(() -> service.getVectorBucket("bucket", REGION), () -> service.deleteVectorBucket("bucket", REGION));+ assertTrue(service.listVectorBuckets(REGION).isEmpty());+ }++ private void overlap(Runnable first, Runnable second) throws Exception {+ Gate gate = new Gate();+ storage.nextWrite.set(gate);+ FutureTask<Void> firstTask = new FutureTask<>(first, null);+ FutureTask<Void> secondTask = new FutureTask<>(second, null);+ Thread firstThread = Thread.ofPlatform().daemon().unstarted(firstTask);+ Thread secondThread = Thread.ofPlatform().daemon().unstarted(secondTask);+ firstThread.start();+ try {+ assertTrue(gate.entered.await(5, TimeUnit.SECONDS), "First writer did not reach storage");+ secondThread.start();+ ThreadStates.awaitBlockedOrDone(secondThread);+ } finally {+ gate.release.countDown();+ }+ firstTask.get(5, TimeUnit.SECONDS);+ secondTask.get(5, TimeUnit.SECONDS);+ }++ private static final class Gate {+ private final CountDownLatch entered = new CountDownLatch(1);+ private final CountDownLatch release = new CountDownLatch(1);+ }++ private static final class ParkingStorage extends InMemoryStorage<String, VectorBucket> {+ private final ObjectMapper mapper = new ObjectMapper().findAndRegisterModules();+ private final boolean copyOnAccess;+ private final AtomicReference<Gate> nextWrite = new AtomicReference<>();++ private ParkingStorage(boolean copyOnAccess) {+ this.copyOnAccess = copyOnAccess;+ }++ @Override+ public Optional<VectorBucket> get(String key) {+ return super.get(key).map(this::copy);+ }++ @Override+ public void put(String key, VectorBucket value) {+ Gate gate = nextWrite.getAndSet(null);+ if (gate != null) {+ gate.entered.countDown();+ try {+ assertTrue(gate.release.await(5, TimeUnit.SECONDS), "Paused write was not released");+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AssertionError("Paused write was interrupted", e);+ }+ }+ super.put(key, copy(value));+ }++ private VectorBucket copy(VectorBucket value) {+ return copyOnAccess ? mapper.convertValue(value, VectorBucket.class) : value;+ }+ }+}