Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3vectors): preserve resource configuration and serialize writes (#5892)

andrehoong-px06bd8c2207ec
diff --git a/docs/services/s3vectors.md b/docs/services/s3vectors.md
index adeb8735e1..8281dc691d 100644
--- a/docs/services/s3vectors.md
+++ b/docs/services/s3vectors.md
@@ -51,3 +51,26 @@ aws s3vectors query-vectors --vector-bucket-name my-vectors --index-name embeddi
Filters are evaluated before similarity ranking and `topK` truncation, so a
nearer vector that does not match the filter cannot consume a result slot.
+
+## Resource configuration
+
+Vector buckets and indexes retain their creation times, encryption settings, and
+creation-time tags. Get and list responses include creation times. `GetIndex`
+returns encryption settings and includes metadata configuration only when
+non-filterable keys are configured. Indexes inherit their bucket's encryption
+settings unless overridden. These fields use the existing S3 Vectors storage backend.
+
+On first read, older stored resources with missing encryption settings receive the
+bucket default, or `AES256` when the bucket has no setting. Missing creation times
+are set to the migration time and saved through the configured storage backend;
+the original creation time is unavailable. Later reads and restarts retain that value.
+
+Index dimensions must be between 1 and 4096. Metadata configuration accepts up to
+10 unique non-filterable keys, each a non-blank string of 1 to 63 characters.
+Queries reject filters on these keys, including filters nested in logical expressions.
+
+Writes to the same bucket are serialized across indexes, vectors, and bucket
+deletion so concurrent changes do not overwrite each other or restore deleted data.
+
+Encryption settings are stored and returned; Floci does not perform KMS encryption.
+Standalone S3 Vectors tagging APIs are not implemented.
diff --git a/pom.xml b/pom.xml
index e448fb2bb2..66f3903112 100644
--- a/pom.xml
+++ b/pom.xml
@@ -330,6 +330,12 @@
</dependency>
<!-- Testing -->
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3vectors</artifactId>
+ <version>${aws.sdk.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>elasticache</artifactId>
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java
index 5f03504ae5..1648cbcd7a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java
@@ -19,7 +19,9 @@ import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import org.jboss.logging.Logger;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import java.util.stream.Collectors;
@Path("/")
@@ -43,7 +45,8 @@ public class S3VectorsController {
@RegisterForReflection
public record CreateVectorBucketRequest(
String vectorBucketName,
- Object encryptionConfiguration
+ Object encryptionConfiguration,
+ Map<String, String> tags
) {}
@RegisterForReflection
@@ -67,6 +70,7 @@ public class S3VectorsController {
public record VectorBucketRepresentation(
String vectorBucketName,
String vectorBucketArn,
+ double creationTime,
Object encryptionConfiguration
) {}
@@ -98,10 +102,14 @@ public class S3VectorsController {
String dataType,
int dimension,
String distanceMetric,
- Object metadataConfiguration,
- Object encryptionConfiguration
+ MetadataConfiguration metadataConfiguration,
+ Object encryptionConfiguration,
+ Map<String, String> tags
) {}
+ @RegisterForReflection
+ public record MetadataConfiguration(List<String> nonFilterableMetadataKeys) {}
+
@RegisterForReflection
public record CreateIndexResponse(
String indexArn
@@ -122,6 +130,7 @@ public class S3VectorsController {
@RegisterForReflection
@JsonInclude(JsonInclude.Include.NON_NULL)
public record IndexRepresentation(
+ double creationTime,
String vectorBucketName,
String indexName,
String indexArn,
@@ -256,7 +265,8 @@ public class S3VectorsController {
@Path("/CreateVectorBucket")
public Response createVectorBucket(@Context HttpHeaders headers, CreateVectorBucketRequest request) {
String region = regionResolver.resolveRegion(headers);
- VectorBucket bucket = service.createVectorBucket(request.vectorBucketName(), request.encryptionConfiguration(), region);
+ VectorBucket bucket = service.createVectorBucket(request.vectorBucketName(), request.encryptionConfiguration(),
+ request.tags() != null ? request.tags() : Map.of(), region);
return Response.ok(new CreateVectorBucketResponse(bucket.getVectorBucketArn())).build();
}
@@ -265,7 +275,8 @@ public class S3VectorsController {
public Response getVectorBucket(@Context HttpHeaders headers, GetVectorBucketRequest request) {
String region = regionResolver.resolveRegion(headers);
VectorBucket bucket = service.getVectorBucket(request.vectorBucketName(), region);
- VectorBucketRepresentation rep = new VectorBucketRepresentation(bucket.getVectorBucketName(), bucket.getVectorBucketArn(), bucket.getEncryptionConfiguration());
+ VectorBucketRepresentation rep = new VectorBucketRepresentation(bucket.getVectorBucketName(),
+ bucket.getVectorBucketArn(), bucket.getCreationTime().toEpochMilli() / 1000.0, bucket.getEncryptionConfiguration());
return Response.ok(new GetVectorBucketResponse(rep)).build();
}
@@ -275,7 +286,8 @@ public class S3VectorsController {
String region = regionResolver.resolveRegion(headers);
List<VectorBucket> buckets = service.listVectorBuckets(region);
List<VectorBucketRepresentation> reps = buckets.stream()
- .map(b -> new VectorBucketRepresentation(b.getVectorBucketName(), b.getVectorBucketArn(), b.getEncryptionConfiguration()))
+ .map(b -> new VectorBucketRepresentation(b.getVectorBucketName(), b.getVectorBucketArn(),
+ b.getCreationTime().toEpochMilli() / 1000.0, b.getEncryptionConfiguration()))
.toList();
return Response.ok(new ListVectorBucketsResponse(reps, null)).build();
}
@@ -298,7 +310,10 @@ public class S3VectorsController {
request.dimension(),
request.dataType() != null ? request.dataType() : "float32",
request.distanceMetric() != null ? request.distanceMetric() : "cosine",
- List.of(),
+ request.metadataConfiguration() != null && request.metadataConfiguration().nonFilterableMetadataKeys() != null
+ ? request.metadataConfiguration().nonFilterableMetadataKeys() : List.of(),
+ request.encryptionConfiguration(),
+ request.tags() != null ? request.tags() : Map.of(),
region
);
return Response.ok(new CreateIndexResponse(index.getIndexArn())).build();
@@ -310,14 +325,16 @@ public class S3VectorsController {
String region = regionResolver.resolveRegion(headers);
VectorIndex index = service.getIndex(request.vectorBucketName(), request.indexName(), region);
IndexRepresentation rep = new IndexRepresentation(
+ index.getCreationTime().toEpochMilli() / 1000.0,
index.getVectorBucketName(),
index.getIndexName(),
index.getIndexArn(),
index.getDataType(),
index.getDimension(),
index.getDistanceMetric(),
- null,
- null
+ index.getNonFilterableMetadataKeys().isEmpty()
+ ? null : new MetadataConfiguration(index.getNonFilterableMetadataKeys()),
+ index.getEncryptionConfiguration()
);
return Response.ok(new GetIndexResponse(rep)).build();
}
@@ -329,6 +346,7 @@ public class S3VectorsController {
List<VectorIndex> indexes = service.listIndexes(request.vectorBucketName(), region);
List<IndexRepresentation> reps = indexes.stream()
.map(i -> new IndexRepresentation(
+ i.getCreationTime().toEpochMilli() / 1000.0,
i.getVectorBucketName(),
i.getIndexName(),
i.getIndexArn(),
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java
index 7d715bd34b..e73a4462c0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java
@@ -4,7 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
-import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.s3vectors.model.VectorBucket;
import io.github.hectorvent.floci.services.s3vectors.model.VectorData;
@@ -14,7 +14,15 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.nio.charset.StandardCharsets;
-import java.util.*;
+import java.time.Clock;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@@ -22,20 +30,36 @@ import java.util.stream.Collectors;
public class S3VectorsService {
private static final Logger LOG = Logger.getLogger(S3VectorsService.class);
- private final StorageBackend<String, VectorBucket> store;
+ private final AccountAwareStorageBackend<VectorBucket> store;
private final RegionResolver regionResolver;
+ private final Clock clock;
+ // Bounded, stable monitors also serialize deletion and recreation of the same bucket.
+ private final Object[] bucketLocks = new Object[256];
@Inject
public S3VectorsService(StorageFactory factory, RegionResolver regionResolver) {
+ this(factory, regionResolver, Clock.systemUTC());
+ }
+
+ S3VectorsService(StorageFactory factory, RegionResolver regionResolver, Clock clock) {
this.store = factory.create("s3vectors", "s3vectors.json",
new TypeReference<Map<String, VectorBucket>>() {});
this.regionResolver = regionResolver;
+ this.clock = clock;
+ for (int i = 0; i < bucketLocks.length; i++) {
+ bucketLocks[i] = new Object();
+ }
}
private String buildStorageKey(String region, String bucketName) {
return region + "::" + bucketName;
}
+ private Object lockFor(String bucketName, String region) {
+ String key = store.accountId() + "/" + buildStorageKey(region, bucketName);
+ return bucketLocks[Math.floorMod(key.hashCode(), bucketLocks.length)];
+ }
+
private String buildBucketArn(String region, String bucketName) {
return AwsArnUtils.Arn.of("s3vectors", region, regionResolver.getAccountId(), "bucket/" + bucketName).toString();
}
@@ -45,68 +69,144 @@ public class S3VectorsService {
}
public VectorBucket createVectorBucket(String bucketName, Object encryptionConfiguration, String region) {
- String key = buildStorageKey(region, bucketName);
- if (store.get(key).isPresent()) {
- throw new AwsException("ConflictException", "The vector bucket " + bucketName + " already exists.", 409);
+ return createVectorBucket(bucketName, encryptionConfiguration, Map.of(), region);
+ }
+
+ public VectorBucket createVectorBucket(String bucketName, Object encryptionConfiguration,
+ Map<String, String> tags, String region) {
+ synchronized (lockFor(bucketName, region)) {
+ String key = buildStorageKey(region, bucketName);
+ if (store.get(key).isPresent()) {
+ throw new AwsException("ConflictException", "The vector bucket " + bucketName + " already exists.", 409);
+ }
+ String arn = buildBucketArn(region, bucketName);
+ VectorBucket bucket = new VectorBucket(bucketName, arn,
+ encryptionConfiguration != null ? encryptionConfiguration : Map.of("sseType", "AES256"));
+ bucket.setCreationTime(clock.instant());
+ bucket.setTags(new ConcurrentHashMap<>(tags));
+ store.put(key, bucket);
+ LOG.infov("Created Vector Bucket: {0} in region {1}", bucketName, region);
+ return bucket;
}
- String arn = buildBucketArn(region, bucketName);
- VectorBucket bucket = new VectorBucket(bucketName, arn, encryptionConfiguration);
- store.put(key, bucket);
- LOG.infov("Created Vector Bucket: {0} in region {1}", bucketName, region);
- return bucket;
}
public VectorBucket getVectorBucket(String bucketName, String region) {
- String key = buildStorageKey(region, bucketName);
- return store.get(key).orElseThrow(() ->
+ return findVectorBucket(bucketName, region).orElseThrow(() ->
new AwsException("NotFoundException", "The vector bucket " + bucketName + " does not exist.", 404));
}
public List<VectorBucket> listVectorBuckets(String region) {
- return store.scan(k -> k.startsWith(region + "::")).stream()
+ String prefix = region + "::";
+ return store.keys().stream().filter(key -> key.startsWith(prefix))
+ .flatMap(key -> findVectorBucket(key.substring(prefix.length()), region).stream())
.sorted(Comparator.comparing(VectorBucket::getVectorBucketName))
.collect(Collectors.toList());
}
+ private Optional<VectorBucket> findVectorBucket(String bucketName, String region) {
+ synchronized (lockFor(bucketName, region)) {
+ String key = buildStorageKey(region, bucketName);
+ return store.get(key).map(bucket -> {
+ if (backfillLegacyMetadata(bucket)) {
+ store.put(key, bucket);
+ }
+ return bucket;
+ });
+ }
+ }
+
+ private boolean backfillLegacyMetadata(VectorBucket bucket) {
+ boolean changed = false;
+ Instant migrationTime = clock.instant();
+ if (bucket.getCreationTime() == null) {
+ bucket.setCreationTime(migrationTime);
+ changed = true;
+ }
+ if (bucket.getEncryptionConfiguration() == null) {
+ bucket.setEncryptionConfiguration(Map.of("sseType", "AES256"));
+ changed = true;
+ }
+ for (VectorIndex index : bucket.getIndexes().values()) {
+ if (index.getCreationTime() == null) {
+ index.setCreationTime(migrationTime);
+ changed = true;
+ }
+ if (index.getEncryptionConfiguration() == null) {
+ index.setEncryptionConfiguration(bucket.getEncryptionConfiguration());
+ changed = true;
+ }
+ }
+ return changed;
+ }
+
public void deleteVectorBucket(String bucketName, String region) {
- String key = buildStorageKey(region, bucketName);
- VectorBucket bucket = store.get(key).orElseThrow(() ->
- new AwsException("NotFoundException", "The vector bucket " + bucketName + " does not exist.", 404));
+ synchronized (lockFor(bucketName, region)) {
+ String key = buildStorageKey(region, bucketName);
+ VectorBucket bucket = store.get(key).orElseThrow(() ->
+ new AwsException("NotFoundException", "The vector bucket " + bucketName + " does not exist.", 404));
- if (!bucket.getIndexes().isEmpty()) {
- throw new AwsException("ConflictException", "The vector bucket " + bucketName + " is not empty.", 409);
+ if (!bucket.getIndexes().isEmpty()) {
+ throw new AwsException("ConflictException", "The vector bucket " + bucketName + " is not empty.", 409);
+ }
+ store.delete(key);
+ LOG.infov("Deleted Vector Bucket: {0}", bucketName);
}
- store.delete(key);
- LOG.infov("Deleted Vector Bucket: {0}", bucketName);
}
public VectorIndex createIndex(String bucketName, String indexName, int dimension, String dataType,
String distanceMetric, List<String> nonFilterableMetadataKeys, String region) {
+ return createIndex(bucketName, indexName, dimension, dataType, distanceMetric,
+ nonFilterableMetadataKeys, null, Map.of(), region);
+ }
+
+ public VectorIndex createIndex(String bucketName, String indexName, int dimension, String dataType,
+ String distanceMetric, List<String> nonFilterableMetadataKeys,
+ Object encryptionConfiguration, Map<String, String> tags, String region) {
+ if (dimension < 1 || dimension > 4096) {
+ throw new AwsException("ValidationException", "Dimension must be between 1 and 4096.", 400);
+ }
if (!"float32".equals(dataType)) {
throw new AwsException("ValidationException", "Unsupported dataType: " + dataType + ". Only float32 is supported.", 400);
}
if (!"cosine".equals(distanceMetric) && !"euclidean".equals(distanceMetric)) {
throw new AwsException("ValidationException", "Unsupported distanceMetric: " + distanceMetric + ". Supported metrics are cosine and euclidean.", 400);
}
-
- VectorBucket bucket = getVectorBucket(bucketName, region);
- if (bucket.getIndexes().containsKey(indexName)) {
- throw new AwsException("ConflictException", "The index " + indexName + " already exists.", 409);
+ if (nonFilterableMetadataKeys == null || nonFilterableMetadataKeys.size() > 10
+ || nonFilterableMetadataKeys.stream().anyMatch(key -> key == null || key.isBlank() || key.length() > 63)
+ || new HashSet<>(nonFilterableMetadataKeys).size() != nonFilterableMetadataKeys.size()) {
+ throw new AwsException("ValidationException",
+ "NonFilterableMetadataKeys must contain at most 10 unique keys of 1 to 63 characters.", 400);
}
- String arn = buildIndexArn(region, bucketName, indexName);
- VectorIndex index = new VectorIndex(indexName, arn, bucketName, dimension, dataType, distanceMetric);
- bucket.getIndexes().put(indexName, index);
- store.put(buildStorageKey(region, bucketName), bucket);
- LOG.infov("Created Index: {0} in bucket {1}", indexName, bucketName);
- return index;
+ synchronized (lockFor(bucketName, region)) {
+ VectorBucket bucket = getVectorBucket(bucketName, region);
+ if (bucket.getIndexes().containsKey(indexName)) {
+ throw new AwsException("ConflictException", "The index " + indexName + " already exists.", 409);
+ }
+
+ String arn = buildIndexArn(region, bucketName, indexName);
+ VectorIndex index = new VectorIndex(indexName, arn, bucketName, dimension, dataType, distanceMetric);
+ index.setCreationTime(clock.instant());
+ index.setNonFilterableMetadataKeys(List.copyOf(nonFilterableMetadataKeys));
+ index.setEncryptionConfiguration(encryptionConfiguration != null
+ ? encryptionConfiguration : bucket.getEncryptionConfiguration());
+ index.setTags(new ConcurrentHashMap<>(tags));
+ bucket.getIndexes().put(indexName, index);
+ store.put(buildStorageKey(region, bucketName), bucket);
+ LOG.infov("Created Index: {0} in bucket {1}", indexName, bucketName);
+ return index;
+ }
}
public VectorIndex getIndex(String bucketName, String indexName, String region) {
- VectorBucket bucket = getVectorBucket(bucketName, region);
+ return getIndex(getVectorBucket(bucketName, region), indexName);
+ }
+
+ private VectorIndex getIndex(VectorBucket bucket, String indexName) {
VectorIndex index = bucket.getIndexes().get(indexName);
if (index == null) {
- throw new AwsException("NotFoundException", "The index " + indexName + " does not exist in bucket " + bucketName + ".", 404);
+ throw new AwsException("NotFoundException",
+ "The index " + indexName + " does not exist in bucket " + bucket.getVectorBucketName() + ".", 404);
}
return index;
}
@@ -119,28 +219,32 @@ public class S3VectorsService {
}
public void deleteIndex(String bucketName, String indexName, String region) {
- VectorBucket bucket = getVectorBucket(bucketName, region);
- if (!bucket.getIndexes().containsKey(indexName)) {
- throw new AwsException("NotFoundException", "The index " + indexName + " does not exist.", 404);
+ synchronized (lockFor(bucketName, region)) {
+ VectorBucket bucket = getVectorBucket(bucketName, region);
+ if (!bucket.getIndexes().containsKey(indexName)) {
+ throw new AwsException("NotFoundException", "The index " + indexName + " does not exist.", 404);
+ }
+ bucket.getIndexes().remove(indexName);
+ store.put(buildStorageKey(region, bucketName), bucket);
+ LOG.infov("Deleted Index: {0}", indexName);
}
- bucket.getIndexes().remove(indexName);
- store.put(buildStorageKey(region, bucketName), bucket);
- LOG.infov("Deleted Index: {0}", indexName);
}
public void putVectors(String bucketName, String indexName, List<VectorData> vectors, String region) {
- VectorBucket bucket = getVectorBucket(bucketName, region);
- VectorIndex index = getIndex(bucketName, indexName, region);
-
- for (VectorData v : vectors) {
- if (v.getData() != null && v.getData().size() != index.getDimension()) {
- throw new AwsException("ValidationException",
- "Vector dimension " + v.getData().size() + " does not match index dimension " + index.getDimension(), 400);
+ synchronized (lockFor(bucketName, region)) {
+ VectorBucket bucket = getVectorBucket(bucketName, region);
+ VectorIndex index = getIndex(bucket, indexName);
+
+ for (VectorData v : vectors) {
+ if (v.getData() != null && v.getData().size() != index.getDimension()) {
+ throw new AwsException("ValidationException",
+ "Vector dimension " + v.getData().size() + " does not match index dimension " + index.getDimension(), 400);
+ }
+ index.getVectors().put(v.getKey(), v);
}
- index.getVectors().put(v.getKey(), v);
+ store.put(buildStorageKey(region, bucketName), bucket);
+ LOG.infov("Put {0} vectors into index {1}", vectors.size(), indexName);
}
- store.put(buildStorageKey(region, bucketName), bucket);
- LOG.infov("Put {0} vectors into index {1}", vectors.size(), indexName);
}
public List<VectorData> getVectors(String bucketName, String indexName, List<String> keys, String region) {
@@ -254,13 +358,15 @@ public class S3VectorsService {
public record ListVectorsResult(List<VectorData> vectors, String nextToken) {}
public void deleteVectors(String bucketName, String indexName, List<String> keys, String region) {
- VectorBucket bucket = getVectorBucket(bucketName, region);
- VectorIndex index = getIndex(bucketName, indexName, region);
- for (String key : keys) {
- index.getVectors().remove(key);
+ synchronized (lockFor(bucketName, region)) {
+ VectorBucket bucket = getVectorBucket(bucketName, region);
+ VectorIndex index = getIndex(bucket, indexName);
+ for (String key : keys) {
+ index.getVectors().remove(key);
+ }
+ store.put(buildStorageKey(region, bucketName), bucket);
+ LOG.infov("Deleted {0} vectors from index {1}", keys.size(), indexName);
}
- store.put(buildStorageKey(region, bucketName), bucket);
- LOG.infov("Deleted {0} vectors from index {1}", keys.size(), indexName);
}
public List<QueryResult> queryVectors(String bucketName, String indexName, List<Float> queryVector,
@@ -272,6 +378,7 @@ public class S3VectorsService {
}
S3VectorsMetadataFilter metadataFilter = S3VectorsMetadataFilter.compile(filter);
+ validateFilterKeys(filter, index.getNonFilterableMetadataKeys());
String metric = index.getDistanceMetric() != null ? index.getDistanceMetric().toLowerCase() : "cosine";
List<QueryResult> results = new ArrayList<>();
@@ -297,6 +404,23 @@ public class S3VectorsService {
return results.stream().limit(topK).collect(Collectors.toList());
}
+ private static void validateFilterKeys(Object filter, List<String> nonFilterableKeys) {
+ if (!(filter instanceof Map<?, ?> clauses)) {
+ return;
+ }
+ for (Map.Entry<?, ?> clause : clauses.entrySet()) {
+ if ("$and".equals(clause.getKey()) || "$or".equals(clause.getKey())) {
+ if (clause.getValue() instanceof List<?> children) {
+ for (Object child : children) {
+ validateFilterKeys(child, nonFilterableKeys);
+ }
+ }
+ } else if (nonFilterableKeys.contains(clause.getKey())) {
+ throw new AwsException("ValidationException", "Cannot filter on non-filterable metadata key: " + clause.getKey(), 400);
+ }
+ }
+ }
+
private double calculateCosineSimilarity(List<Float> v1, List<Float> v2) {
if (v1 == null || v2 == null || v1.size() != v2.size()) return 0.0;
double dotProduct = 0.0;
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorBucket.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorBucket.java
index 6fd3e3a9f6..2111268daa 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorBucket.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorBucket.java
@@ -2,11 +2,14 @@ package io.github.hectorvent.floci.services.s3vectors.model;
import io.quarkus.runtime.annotations.RegisterForReflection;
+import java.time.Instant;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@RegisterForReflection
public class VectorBucket {
+ private Instant creationTime;
+ private Map<String, String> tags = new ConcurrentHashMap<>();
private String vectorBucketName;
private String vectorBucketArn;
private Object encryptionConfiguration;
@@ -20,6 +23,12 @@ public class VectorBucket {
this.encryptionConfiguration = encryptionConfiguration;
}
+ public Instant getCreationTime() { return creationTime; }
+ public void setCreationTime(Instant creationTime) { this.creationTime = creationTime; }
+
+ public Map<String, String> getTags() { return tags; }
+ public void setTags(Map<String, String> tags) { this.tags = tags; }
+
public String getVectorBucketName() { return vectorBucketName; }
public void setVectorBucketName(String vectorBucketName) { this.vectorBucketName = vectorBucketName; }
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorIndex.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorIndex.java
index 2742d0f0af..69aa868060 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorIndex.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/model/VectorIndex.java
@@ -2,14 +2,20 @@ package io.github.hectorvent.floci.services.s3vectors.model;
import io.quarkus.runtime.annotations.RegisterForReflection;
+import java.time.Instant;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@RegisterForReflection
public class VectorIndex {
+ private Instant creationTime;
+ private Map<String, String> tags = new ConcurrentHashMap<>();
private String indexName;
private String indexArn;
private String vectorBucketName;
+ private Object encryptionConfiguration;
+ private List<String> nonFilterableMetadataKeys = List.of();
private int dimension;
private String dataType;
private String distanceMetric;
@@ -26,6 +32,12 @@ public class VectorIndex {
this.distanceMetric = distanceMetric;
}
+ public Instant getCreationTime() { return creationTime; }
+ public void setCreationTime(Instant creationTime) { this.creationTime = creationTime; }
+
+ public Map<String, String> getTags() { return tags; }
+ public void setTags(Map<String, String> tags) { this.tags = tags; }
+
public String getIndexName() { return indexName; }
public void setIndexName(String indexName) { this.indexName = indexName; }
@@ -35,6 +47,12 @@ public class VectorIndex {
public String getVectorBucketName() { return vectorBucketName; }
public void setVectorBucketName(String vectorBucketName) { this.vectorBucketName = vectorBucketName; }
+ public Object getEncryptionConfiguration() { return encryptionConfiguration; }
+ public void setEncryptionConfiguration(Object encryptionConfiguration) { this.encryptionConfiguration = encryptionConfiguration; }
+
+ public List<String> getNonFilterableMetadataKeys() { return nonFilterableMetadataKeys; }
+ public void setNonFilterableMetadataKeys(List<String> keys) { this.nonFilterableMetadataKeys = keys; }
+
public int getDimension() { return dimension; }
public void setDimension(int dimension) { this.dimension = dimension; }
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsLegacySdkIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsLegacySdkIntegrationTest.java
new file mode 100644
index 0000000000..782acff4cb
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsLegacySdkIntegrationTest.java
@@ -0,0 +1,86 @@
+package io.github.hectorvent.floci.services.s3vectors;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.s3vectors.model.VectorBucket;
+import io.github.hectorvent.floci.services.s3vectors.model.VectorIndex;
+import io.quarkus.test.common.http.TestHTTPResource;
+import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3vectors.S3VectorsClient;
+import software.amazon.awssdk.services.s3vectors.model.GetVectorBucketResponse;
+import software.amazon.awssdk.services.s3vectors.model.Index;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+@QuarkusTest
+class S3VectorsLegacySdkIntegrationTest {
+ @TestHTTPResource
+ URI endpoint;
+
+ @Inject
+ S3VectorsService service;
+
+ @Inject
+ StorageFactory factory;
+
+ @Inject
+ ObjectMapper mapper;
+
+ @Test
+ void legacyBucketAndIndexMetadataIsAvailableThroughSdkGetAndList() throws Exception {
+ String region = "us-east-1";
+ String name = "legacy-vectors-" + UUID.randomUUID().toString().substring(0, 8);
+ service.createVectorBucket(name, null, region);
+ service.createIndex(name, "old", 1, "float32", "cosine", List.of(), region);
+ ObjectNode legacyJson = mapper.valueToTree(service.getVectorBucket(name, region));
+ legacyJson.remove(List.of("creationTime", "encryptionConfiguration"));
+ ObjectNode legacyIndex = (ObjectNode) legacyJson.path("indexes").path("old");
+ legacyIndex.remove(List.of("creationTime", "encryptionConfiguration"));
+ AccountAwareStorageBackend<VectorBucket> store = factory.create("s3vectors", "s3vectors.json",
+ new TypeReference<Map<String, VectorBucket>>() {});
+ store.put(region + "::" + name, mapper.treeToValue(legacyJson, VectorBucket.class));
+
+ try (S3VectorsClient vectors = S3VectorsClient.builder().endpointOverride(endpoint).region(Region.of(region))
+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("test", "test")))
+ .httpClientBuilder(UrlConnectionHttpClient.builder()).build()) {
+ try {
+ // Creating an index must also resolve the legacy bucket's missing encryption default.
+ vectors.createIndex(request -> request.vectorBucketName(name).indexName("new")
+ .dimension(1).dataType("float32").distanceMetric("cosine"));
+ GetVectorBucketResponse bucket = vectors.getVectorBucket(request -> request.vectorBucketName(name));
+ assertEquals("AES256", bucket.vectorBucket().encryptionConfiguration().sseTypeAsString());
+ assertNotNull(bucket.vectorBucket().creationTime());
+ assertEquals(bucket.vectorBucket().creationTime(), vectors.listVectorBuckets(request -> request.maxResults(100))
+ .vectorBuckets().stream().filter(item -> name.equals(item.vectorBucketName()))
+ .findFirst().orElseThrow().creationTime());
+ for (String indexName : List.of("old", "new")) {
+ Index index = vectors.getIndex(request -> request.vectorBucketName(name).indexName(indexName)).index();
+ assertEquals("AES256", index.encryptionConfiguration().sseTypeAsString());
+ assertNotNull(index.creationTime());
+ assertEquals(index.creationTime(), vectors.listIndexes(request -> request.vectorBucketName(name))
+ .indexes().stream().filter(item -> indexName.equals(item.indexName()))
+ .findFirst().orElseThrow().creationTime());
+ }
+ } finally {
+ for (VectorIndex index : service.listIndexes(name, region)) {
+ vectors.deleteIndex(request -> request.vectorBucketName(name).indexName(index.getIndexName()));
+ }
+ vectors.deleteVectorBucket(request -> request.vectorBucketName(name));
+ }
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsPersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsPersistenceTest.java
new file mode 100644
index 0000000000..34d2e84fb6
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsPersistenceTest.java
@@ -0,0 +1,133 @@
+package io.github.hectorvent.floci.services.s3vectors;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.HybridStorage;
+import io.github.hectorvent.floci.core.storage.PersistentStorage;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.core.storage.WalStorage;
+import io.github.hectorvent.floci.services.s3vectors.model.VectorBucket;
+import io.github.hectorvent.floci.services.s3vectors.model.VectorIndex;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+class S3VectorsPersistenceTest {
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT = "000000000000";
+ private static final String KEY = ACCOUNT + "/" + REGION + "::legacy";
+ private static final Map<String, String> DEFAULT_ENCRYPTION = Map.of("sseType", "AES256");
+ private static final Instant KNOWN_TIME = Instant.parse("2026-01-01T00:00:00Z");
+ private static final Instant MIGRATION_TIME = Instant.parse("2026-02-01T00:00:00Z");
+
+ @ParameterizedTest
+ @CsvSource({"persistent,false", "persistent,true", "hybrid,false", "hybrid,true", "wal,false", "wal,true"})
+ void legacyMetadataIsSavedOnceAndSurvivesReload(String mode, boolean listFirst, @TempDir Path directory)
+ throws Exception {
+ Path snapshot = directory.resolve("s3vectors.json");
+ Files.writeString(snapshot, """
+ {
+ "000000000000/us-east-1::legacy": {
+ "vectorBucketName": "legacy",
+ "encryptionConfiguration": null,
+ "indexes": {
+ "old": {"indexName": "old", "dimension": 1, "dataType": "float32", "distanceMetric": "cosine"},
+ "configured": {
+ "indexName": "configured",
+ "creationTime": "2026-01-01T00:00:00Z",
+ "encryptionConfiguration": {"sseType": "aws:kms", "kmsKeyArn": "key"}
+ }
+ }
+ }
+ }
+ """);
+ StorageBackend<String, VectorBucket> first = open(mode, snapshot);
+ try {
+ VectorBucket legacy = first.get(KEY).orElseThrow();
+ assertNull(legacy.getCreationTime());
+ assertNull(legacy.getIndexes().get("old").getCreationTime());
+ S3VectorsService service = service(first, Clock.fixed(MIGRATION_TIME, ZoneOffset.UTC));
+ VectorBucket bucket = listFirst ? service.listVectorBuckets(REGION).getFirst()
+ : service.getVectorBucket("legacy", REGION);
+ VectorIndex index = service.getIndex("legacy", "old", REGION);
+ assertEquals(MIGRATION_TIME, bucket.getCreationTime());
+ assertEquals(MIGRATION_TIME, index.getCreationTime());
+ assertEquals(DEFAULT_ENCRYPTION, bucket.getEncryptionConfiguration());
+ assertEquals(DEFAULT_ENCRYPTION, index.getEncryptionConfiguration());
+ assertEquals(KNOWN_TIME, bucket.getIndexes().get("configured").getCreationTime());
+ assertEquals(Map.of("sseType", "aws:kms", "kmsKeyArn", "key"),
+ bucket.getIndexes().get("configured").getEncryptionConfiguration());
+ assertEquals(MIGRATION_TIME, service.listVectorBuckets(REGION).getFirst().getCreationTime());
+ assertEquals(MIGRATION_TIME, service.listIndexes("legacy", REGION).stream()
+ .filter(item -> "old".equals(item.getIndexName())).findFirst().orElseThrow().getCreationTime());
+ verify(first, times(1)).put(eq(KEY), any());
+
+ VectorIndex created = service.createIndex("legacy", "new", 1, "float32", "cosine", List.of(), REGION);
+ assertEquals(DEFAULT_ENCRYPTION, created.getEncryptionConfiguration());
+ assertEquals(MIGRATION_TIME, created.getCreationTime());
+ } finally {
+ close(first);
+ }
+
+ StorageBackend<String, VectorBucket> second = open(mode, snapshot);
+ try {
+ VectorBucket stored = second.get(KEY).orElseThrow();
+ assertEquals(MIGRATION_TIME, stored.getCreationTime());
+ assertEquals(MIGRATION_TIME, stored.getIndexes().get("old").getCreationTime());
+ assertEquals(DEFAULT_ENCRYPTION, stored.getEncryptionConfiguration());
+ S3VectorsService restarted = service(second, Clock.fixed(MIGRATION_TIME.plusSeconds(86_400), ZoneOffset.UTC));
+ assertEquals(MIGRATION_TIME, restarted.getVectorBucket("legacy", REGION).getCreationTime());
+ assertEquals(MIGRATION_TIME, restarted.listVectorBuckets(REGION).getFirst().getCreationTime());
+ assertEquals(MIGRATION_TIME, restarted.getIndex("legacy", "old", REGION).getCreationTime());
+ assertEquals(MIGRATION_TIME, restarted.getIndex("legacy", "new", REGION).getCreationTime());
+ assertEquals(KNOWN_TIME, restarted.getIndex("legacy", "configured", REGION).getCreationTime());
+ assertEquals(DEFAULT_ENCRYPTION, restarted.getIndex("legacy", "new", REGION).getEncryptionConfiguration());
+ assertEquals(3, restarted.listIndexes("legacy", REGION).size());
+ verify(second, never()).put(anyString(), any());
+ } finally {
+ close(second);
+ }
+ }
+
+ private static S3VectorsService service(StorageBackend<String, VectorBucket> backend, Clock clock) {
+ StorageFactory factory = mock(StorageFactory.class);
+ when(factory.<VectorBucket>create(anyString(), anyString(), any()))
+ .thenReturn(new AccountAwareStorageBackend<>(backend, null, ACCOUNT));
+ return new S3VectorsService(factory, new RegionResolver(REGION, ACCOUNT), clock);
+ }
+
+ private static StorageBackend<String, VectorBucket> open(String mode, Path snapshot) {
+ TypeReference<Map<String, VectorBucket>> type = new TypeReference<>() {};
+ StorageBackend<String, VectorBucket> backend = spy(switch (mode) {
+ case "persistent" -> new PersistentStorage<>(snapshot, type);
+ case "hybrid" -> new HybridStorage<>(snapshot, type, 60_000);
+ case "wal" -> new WalStorage<>(snapshot, snapshot.resolveSibling("s3vectors.wal"), type, 60_000);
+ default -> throw new IllegalArgumentException(mode);
+ });
+ backend.load();
+ return backend;
+ }
+
+ private static void close(StorageBackend<String, VectorBucket> backend) {
+ if (backend instanceof HybridStorage<?, ?> hybrid) {
+ hybrid.shutdown();
+ } else if (backend instanceof WalStorage<?, ?> wal) {
+ wal.shutdown();
+ } else {
+ backend.flush();
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsSdkIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsSdkIntegrationTest.java
new file mode 100644
index 0000000000..4cd8c7926a
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsSdkIntegrationTest.java
@@ -0,0 +1,236 @@
+package io.github.hectorvent.floci.services.s3vectors;
+
+import io.github.hectorvent.floci.services.s3vectors.model.VectorIndex;
+import io.quarkus.test.common.http.TestHTTPResource;
+import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.document.Document;
+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3vectors.S3VectorsClient;
+import software.amazon.awssdk.services.s3vectors.model.Index;
+import software.amazon.awssdk.services.s3vectors.model.PutInputVector;
+import software.amazon.awssdk.services.s3vectors.model.QueryVectorsResponse;
+import software.amazon.awssdk.services.s3vectors.model.ValidationException;
+import software.amazon.awssdk.services.s3vectors.model.VectorBucket;
+import software.amazon.awssdk.services.s3vectors.model.VectorData;
+
+import java.net.URI;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Stream;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.*;
+
+@QuarkusTest
+class S3VectorsSdkIntegrationTest {
+
+ @TestHTTPResource
+ URI endpoint;
+
+ @Inject
+ S3VectorsService service;
+
+ @Test
+ void sdkCreatedResourcesPreserveConfigurationAndRejectNonFilterableQueries() {
+ String name = "sdk-vectors-" + UUID.randomUUID().toString().substring(0, 8);
+ try (S3VectorsClient vectors = vectors("us-east-1")) {
+ vectors.createVectorBucket(request -> request.vectorBucketName(name)
+ .encryptionConfiguration(encryption -> encryption.sseType("AES256"))
+ .tags(Map.of("team", "search")));
+ try {
+ vectors.createIndex(request -> request.vectorBucketName(name).indexName("index")
+ .dataType("float32").dimension(3).distanceMetric("cosine")
+ .metadataConfiguration(metadata -> metadata.nonFilterableMetadataKeys("text"))
+ .tags(Map.of("team", "search")));
+ Index index = vectors.getIndex(request -> request.vectorBucketName(name).indexName("index")).index();
+ VectorBucket bucket = vectors.getVectorBucket(request -> request.vectorBucketName(name)).vectorBucket();
+ assertNotNull(bucket.creationTime());
+ assertNotNull(index.creationTime());
+ assertEquals(service.getVectorBucket(name, "us-east-1").getCreationTime().toEpochMilli(),
+ bucket.creationTime().toEpochMilli());
+ assertEquals(service.getIndex(name, "index", "us-east-1").getCreationTime().toEpochMilli(),
+ index.creationTime().toEpochMilli());
+ assertEquals(bucket.creationTime(), vectors.listVectorBuckets(request -> request.maxResults(100))
+ .vectorBuckets().stream()
+ .filter(item -> name.equals(item.vectorBucketName())).findFirst().orElseThrow().creationTime());
+ assertEquals(index.creationTime(), vectors.listIndexes(request -> request.vectorBucketName(name))
+ .indexes().getFirst().creationTime());
+ assertEquals(List.of("text"), index.metadataConfiguration().nonFilterableMetadataKeys());
+ assertEquals("AES256", index.encryptionConfiguration().sseTypeAsString());
+ assertEquals(Map.of("team", "search"), service.getVectorBucket(name, "us-east-1").getTags());
+ assertEquals(Map.of("team", "search"), service.getIndex(name, "index", "us-east-1").getTags());
+ assertThrows(ValidationException.class, () -> vectors.queryVectors(request -> request
+ .vectorBucketName(name).indexName("index").topK(1)
+ .queryVector(VectorData.builder().float32(1.0f, 0.0f, 0.0f).build())
+ .filter(Document.fromMap(Map.of("text", Document.fromString("hello"))))));
+ } finally {
+ for (String index : service.listIndexes(name, "us-east-1").stream()
+ .map(VectorIndex::getIndexName).toList()) {
+ vectors.deleteIndex(request -> request.vectorBucketName(name).indexName(index));
+ }
+ vectors.deleteVectorBucket(request -> request.vectorBucketName(name));
+ }
+ }
+ }
+
+ static Stream<Arguments> invalidIndexSettings() {
+ return Stream.of(
+ Arguments.of("float64", "cosine", List.of(), "dataType"),
+ Arguments.of("float32", "manhattan", List.of(), "distanceMetric"),
+ Arguments.of("float32", "cosine", List.of(""), "NonFilterableMetadataKeys"),
+ Arguments.of("float32", "cosine", List.of(" "), "NonFilterableMetadataKeys"),
+ Arguments.of("float32", "cosine", Collections.singletonList(null), "NonFilterableMetadataKeys"),
+ Arguments.of("float32", "cosine", List.of("a".repeat(64)), "NonFilterableMetadataKeys"),
+ Arguments.of("float32", "cosine", List.of("text", "text"), "NonFilterableMetadataKeys"),
+ Arguments.of("float32", "cosine", Collections.nCopies(11, "key"), "NonFilterableMetadataKeys"));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"$and", "$or"})
+ void logicalOperatorsAreNotTreatedAsNonFilterableFields(String operator) {
+ String name = "logical-vectors-" + UUID.randomUUID().toString().substring(0, 8);
+ try (S3VectorsClient vectors = vectors("us-east-1")) {
+ vectors.createVectorBucket(request -> request.vectorBucketName(name));
+ try {
+ vectors.createIndex(request -> request.vectorBucketName(name).indexName("index")
+ .dataType("float32").dimension(1).distanceMetric("cosine")
+ .metadataConfiguration(metadata -> metadata.nonFilterableMetadataKeys("$and", "$or", "text")));
+ vectors.putVectors(request -> request.vectorBucketName(name).indexName("index")
+ .vectors(PutInputVector.builder().key("match")
+ .data(VectorData.builder().float32(1.0f).build())
+ .metadata(Document.fromMap(Map.of("category", Document.fromString("x")))).build()));
+ Document filter = Document.fromMap(Map.of(operator, Document.fromList(List.of(
+ Document.fromMap(Map.of("category", Document.fromString("x")))))));
+ QueryVectorsResponse result = vectors.queryVectors(request -> request.vectorBucketName(name)
+ .indexName("index").queryVector(VectorData.builder().float32(1.0f).build()).topK(1).filter(filter));
+ assertEquals(List.of("match"), result.vectors().stream().map(item -> item.key()).toList());
+
+ Document forbidden = Document.fromMap(Map.of(operator, Document.fromList(List.of(
+ Document.fromMap(Map.of("$and", Document.fromList(List.of(
+ Document.fromMap(Map.of("text", Document.fromString("hidden")))))))))));
+ ValidationException error = assertThrows(ValidationException.class, () -> vectors.queryVectors(request -> request
+ .vectorBucketName(name).indexName("index").queryVector(VectorData.builder().float32(1.0f).build())
+ .topK(1).filter(forbidden)));
+ assertEquals(400, error.statusCode());
+ assertTrue(error.getMessage().contains("text"));
+ } finally {
+ for (VectorIndex index : service.listIndexes(name, "us-east-1")) {
+ vectors.deleteIndex(request -> request.vectorBucketName(name).indexName(index.getIndexName()));
+ }
+ vectors.deleteVectorBucket(request -> request.vectorBucketName(name));
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void configurationAppearsOnlyInGetIndexAndEmptyMetadataIsOmitted(boolean configured) {
+ String name = "response-vectors-" + UUID.randomUUID().toString().substring(0, 8);
+ try (S3VectorsClient vectors = vectors("us-east-1")) {
+ vectors.createVectorBucket(request -> request.vectorBucketName(name));
+ try {
+ vectors.createIndex(request -> {
+ request.vectorBucketName(name).indexName("index").dimension(3)
+ .dataType("float32").distanceMetric("cosine");
+ if (configured) {
+ request.metadataConfiguration(metadata -> metadata.nonFilterableMetadataKeys("text"));
+ }
+ });
+ Index index = vectors.getIndex(request -> request.vectorBucketName(name).indexName("index")).index();
+ assertEquals("AES256", index.encryptionConfiguration().sseTypeAsString());
+ if (configured) {
+ assertEquals(List.of("text"), index.metadataConfiguration().nonFilterableMetadataKeys());
+ } else {
+ assertNull(index.metadataConfiguration());
+ }
+
+ Map<String, Object> detail = given().baseUri(endpoint.toString()).contentType("application/json")
+ .body(Map.of("vectorBucketName", name, "indexName", "index"))
+ .post("/GetIndex").then().statusCode(200).extract().path("index");
+ assertEquals(configured, detail.containsKey("metadataConfiguration"));
+ assertTrue(detail.containsKey("encryptionConfiguration"));
+ // SDK clients ignore unknown fields, so check the wire response as well.
+ Map<String, Object> summary = given().baseUri(endpoint.toString()).contentType("application/json")
+ .body(Map.of("vectorBucketName", name))
+ .post("/ListIndexes").then().statusCode(200).extract().path("indexes[0]");
+ assertEquals(index.indexArn(), summary.get("indexArn"));
+ assertFalse(summary.containsKey("metadataConfiguration"));
+ assertFalse(summary.containsKey("encryptionConfiguration"));
+ } finally {
+ for (VectorIndex index : service.listIndexes(name, "us-east-1")) {
+ vectors.deleteIndex(request -> request.vectorBucketName(name).indexName(index.getIndexName()));
+ }
+ vectors.deleteVectorBucket(request -> request.vectorBucketName(name));
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("invalidIndexSettings")
+ void invalidIndexSettingsReturnValidationErrors(String dataType, String distanceMetric,
+ List<String> keys, String errorMember) {
+ String name = "invalid-vectors-" + UUID.randomUUID().toString().substring(0, 8);
+ try (S3VectorsClient vectors = vectors("us-east-1")) {
+ vectors.createVectorBucket(request -> request.vectorBucketName(name));
+ try {
+ ValidationException error = assertThrows(ValidationException.class,
+ () -> vectors.createIndex(request -> request.vectorBucketName(name).indexName("index")
+ .dimension(3).dataType(dataType).distanceMetric(distanceMetric)
+ .metadataConfiguration(metadata -> metadata.nonFilterableMetadataKeys(keys))));
+ assertEquals(400, error.statusCode());
+ assertTrue(error.getMessage().contains(errorMember), error.getMessage());
+ assertTrue(vectors.listIndexes(request -> request.vectorBucketName(name)).indexes().isEmpty());
+ } finally {
+ for (VectorIndex index : service.listIndexes(name, "us-east-1")) {
+ vectors.deleteIndex(request -> request.vectorBucketName(name).indexName(index.getIndexName()));
+ }
+ vectors.deleteVectorBucket(request -> request.vectorBucketName(name));
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {0, 1, 4096, 4097})
+ void dimensionBoundariesAreEnforcedThroughTheSdk(int dimension) {
+ String name = "dimension-vectors-" + UUID.randomUUID().toString().substring(0, 8);
+ try (S3VectorsClient vectors = vectors("us-east-1")) {
+ vectors.createVectorBucket(request -> request.vectorBucketName(name));
+ try {
+ if (dimension == 0 || dimension == 4097) {
+ ValidationException error = assertThrows(ValidationException.class,
+ () -> vectors.createIndex(request -> request.vectorBucketName(name).indexName("index")
+ .dimension(dimension).dataType("float32").distanceMetric("cosine")));
+ assertEquals(400, error.statusCode());
+ assertTrue(vectors.listIndexes(request -> request.vectorBucketName(name)).indexes().isEmpty());
+ } else {
+ vectors.createIndex(request -> request.vectorBucketName(name).indexName("index")
+ .dimension(dimension).dataType("float32").distanceMetric("cosine"));
+ assertEquals(dimension, vectors.getIndex(request -> request.vectorBucketName(name)
+ .indexName("index")).index().dimension());
+ }
+ } finally {
+ for (VectorIndex index : service.listIndexes(name, "us-east-1")) {
+ vectors.deleteIndex(request -> request.vectorBucketName(name).indexName(index.getIndexName()));
+ }
+ vectors.deleteVectorBucket(request -> request.vectorBucketName(name));
+ }
+ }
+ }
+
+ private S3VectorsClient vectors(String region) {
+ return S3VectorsClient.builder().endpointOverride(endpoint).region(Region.of(region))
+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("test", "test")))
+ .httpClientBuilder(UrlConnectionHttpClient.builder()).build();
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsServiceTest.java
new file mode 100644
index 0000000000..660af4a2ea
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsServiceTest.java
@@ -0,0 +1,114 @@
+package io.github.hectorvent.floci.services.s3vectors;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.s3vectors.model.VectorBucket;
+import io.github.hectorvent.floci.services.s3vectors.model.VectorIndex;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class S3VectorsServiceTest {
+
+ private static final String REGION = "us-east-1";
+ private static final Instant CREATION_TIME = Instant.parse("2026-02-01T00:00:00Z");
+ private S3VectorsService service;
+
+ @BeforeEach
+ void setUp() {
+ StorageFactory factory = new StorageFactory(null, null) {
+ @Override
+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return AccountAwareStorageBackend.inMemory("000000000000");
+ }
+ };
+ service = new S3VectorsService(factory, new RegionResolver(REGION, "000000000000"),
+ Clock.fixed(CREATION_TIME, ZoneOffset.UTC));
+ }
+
+ @Test
+ void persistsCreationTimesConfigurationAndCreationTags() throws Exception {
+ Map<String, String> encryption = Map.of("sseType", "aws:kms", "kmsKeyArn", "key");
+ VectorBucket bucket = service.createVectorBucket("bucket", encryption, Map.of("old", "value"), REGION);
+ VectorIndex index = service.createIndex("bucket", "index", 3, "float32", "cosine", List.of("text"),
+ null, Map.of("old", "value"), REGION);
+ assertEquals(CREATION_TIME, bucket.getCreationTime());
+ assertEquals(CREATION_TIME, index.getCreationTime());
+
+ ObjectMapper mapper = new ObjectMapper().findAndRegisterModules();
+ VectorBucket restored = mapper.readValue(mapper.writeValueAsString(bucket), VectorBucket.class);
+ VectorIndex restoredIndex = restored.getIndexes().get("index");
+ assertEquals(bucket.getCreationTime(), restored.getCreationTime());
+ assertEquals(index.getCreationTime(), restoredIndex.getCreationTime());
+ assertEquals(encryption, restoredIndex.getEncryptionConfiguration());
+ assertEquals(List.of("text"), restoredIndex.getNonFilterableMetadataKeys());
+ assertEquals(Map.of("old", "value"), restored.getTags());
+ assertEquals(Map.of("old", "value"), restoredIndex.getTags());
+ }
+
+ @Test
+ void indexEncryptionCanOverrideTheBucketDefault() {
+ VectorBucket bucket = service.createVectorBucket("bucket", null, REGION);
+ assertEquals(Map.of("sseType", "AES256"), bucket.getEncryptionConfiguration());
+ Map<String, String> encryption = Map.of("sseType", "aws:kms", "kmsKeyArn", "key");
+ VectorIndex index = service.createIndex("bucket", "index", 3, "float32", "cosine", List.of(),
+ encryption, Map.of(), REGION);
+ assertEquals(encryption, index.getEncryptionConfiguration());
+ }
+
+ @Test
+ void nonFilterableMetadataKeysAreRejectedEvenInsideLogicalFilters() {
+ service.createVectorBucket("bucket", null, REGION);
+ service.createIndex("bucket", "index", 3, "float32", "cosine", List.of("text"), REGION);
+ for (Object filter : List.of(Map.of("text", "hello"), Map.of("$and", List.of(Map.of("text", "hello"))))) {
+ AwsException error = assertThrows(AwsException.class,
+ () -> service.queryVectors("bucket", "index", List.of(1.0f, 0.0f, 0.0f), filter, 1, REGION));
+ assertEquals("ValidationException", error.getErrorCode());
+ }
+ assertEquals(List.of(), service.queryVectors("bucket", "index", List.of(1.0f, 0.0f, 0.0f),
+ Map.of("category", "hello"), 1, REGION));
+ }
+
+ @Test
+ void rejectsInvalidIndexDimensions() {
+ service.createVectorBucket("bucket", null, REGION);
+ for (int dimension : List.of(0, 4097)) {
+ assertThrows(AwsException.class,
+ () -> service.createIndex("bucket", "index", dimension, "float32", "cosine", List.of(), REGION));
+ }
+ assertTrue(service.listIndexes("bucket", REGION).isEmpty());
+ }
+
+ @Test
+ void invalidMetadataKeysFailBeforeCreatingAnIndex() {
+ service.createVectorBucket("bucket", null, REGION);
+ for (List<String> keys : List.of(List.of(""), List.of(" "), List.of("a".repeat(64)),
+ Collections.nCopies(11, "key"), List.of("text", "text"), Collections.<String>singletonList(null))) {
+ AwsException error = assertThrows(AwsException.class,
+ () -> service.createIndex("bucket", "index", 3, "float32", "cosine", keys, REGION));
+ assertEquals("ValidationException", error.getErrorCode());
+ }
+ assertTrue(service.listIndexes("bucket", REGION).isEmpty());
+ }
+
+ @Test
+ void metadataKeyLimitsAreInclusive() {
+ service.createVectorBucket("bucket", null, REGION);
+ List<String> keys = List.of("a".repeat(63), "b", "c", "d", "e", "f", "g", "h", "i", "j");
+ VectorIndex index = service.createIndex("bucket", "index", 3, "float32", "euclidean", keys, REGION);
+ assertEquals(keys, index.getNonFilterableMetadataKeys());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsWriteLockTest.java b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsWriteLockTest.java
new file mode 100644
index 0000000000..e9f9bb7e90
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsWriteLockTest.java
@@ -0,0 +1,179 @@
+package io.github.hectorvent.floci.services.s3vectors;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.s3vectors.model.VectorBucket;
+import io.github.hectorvent.floci.services.s3vectors.model.VectorData;
+import io.github.hectorvent.floci.testing.ThreadStates;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+
+class S3VectorsWriteLockTest {
+
+ private static final String REGION = "us-east-1";
+ private ParkingStorage storage;
+ private S3VectorsService service;
+
+ private void setUp(boolean copyOnAccess) {
+ storage = new ParkingStorage(copyOnAccess);
+ StorageFactory factory = mock(StorageFactory.class);
+ when(factory.<VectorBucket>create(anyString(), anyString(), any()))
+ .thenReturn(new AccountAwareStorageBackend<>(storage, null, "000000000000"));
+ service = new S3VectorsService(factory, new RegionResolver(REGION, "000000000000"));
+ service.createVectorBucket("bucket", null, REGION);
+ }
+
+ private void createIndex() {
+ service.createIndex("bucket", "index", 1, "float32", "cosine", List.of(), REGION);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void vectorWritesPreserveConcurrentIndexCreation(boolean copyOnAccess) throws Exception {
+ setUp(copyOnAccess);
+ createIndex();
+ overlap(() -> putVector("first"), () -> service.createIndex("bucket", "second", 1,
+ "float32", "cosine", List.of(), REGION));
+ assertEquals("second", service.getIndex("bucket", "second", REGION).getIndexName());
+ assertEquals(1, service.getVectors("bucket", "index", List.of("first"), REGION).size());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void concurrentVectorWritesPreserveBothVectors(boolean copyOnAccess) throws Exception {
+ setUp(copyOnAccess);
+ createIndex();
+ overlap(() -> putVector("first"), () -> putVector("second"));
+ assertEquals(2, service.getVectors("bucket", "index", List.of("first", "second"), REGION).size());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void vectorWritesDoNotReviveDeletedVectorsOrIndexes(boolean copyOnAccess) throws Exception {
+ setUp(copyOnAccess);
+ createIndex();
+ putVector("old");
+ overlap(() -> putVector("new"), () -> service.deleteVectors("bucket", "index", List.of("old"), REGION));
+ assertTrue(service.getVectors("bucket", "index", List.of("old"), REGION).isEmpty());
+ assertEquals(1, service.getVectors("bucket", "index", List.of("new"), REGION).size());
+
+ overlap(() -> putVector("new"), () -> service.deleteIndex("bucket", "index", REGION));
+ AwsException error = assertThrows(AwsException.class, () -> service.getIndex("bucket", "index", REGION));
+ assertEquals("NotFoundException", error.getErrorCode());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void indexDeletionCannotResurrectADeletedOrRecreatedBucket(boolean copyOnAccess) throws Exception {
+ setUp(copyOnAccess);
+ createIndex();
+ overlap(() -> service.deleteIndex("bucket", "index", REGION), () -> service.deleteVectorBucket("bucket", REGION));
+ assertThrows(AwsException.class, () -> service.getVectorBucket("bucket", REGION));
+
+ service.createVectorBucket("bucket", null, REGION);
+ createIndex();
+ overlap(() -> service.deleteIndex("bucket", "index", REGION), () -> {
+ service.deleteVectorBucket("bucket", REGION);
+ service.createVectorBucket("bucket", null, Map.of("new", "tag"), REGION);
+ });
+ assertEquals(Map.of("new", "tag"), service.getVectorBucket("bucket", REGION).getTags());
+ }
+
+ private void putVector(String key) {
+ service.putVectors("bucket", "index", List.of(new VectorData(key, List.of(1.0f), Map.of())), REGION);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void legacyMetadataMigrationDoesNotOverwriteConcurrentWrites(boolean copyOnAccess) throws Exception {
+ setUp(copyOnAccess);
+ VectorBucket legacy = new VectorBucket("bucket", "bucket-arn", null);
+ storage.put("000000000000/" + REGION + "::bucket", legacy);
+ overlap(() -> service.listVectorBuckets(REGION), this::createIndex);
+ assertNotNull(service.getVectorBucket("bucket", REGION).getCreationTime());
+ assertEquals(Map.of("sseType", "AES256"), service.getIndex("bucket", "index", REGION).getEncryptionConfiguration());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void legacyMetadataMigrationDoesNotReviveADeletedBucket(boolean copyOnAccess) throws Exception {
+ setUp(copyOnAccess);
+ storage.put("000000000000/" + REGION + "::bucket", new VectorBucket("bucket", "bucket-arn", null));
+ overlap(() -> service.getVectorBucket("bucket", REGION), () -> service.deleteVectorBucket("bucket", REGION));
+ assertTrue(service.listVectorBuckets(REGION).isEmpty());
+ }
+
+ private void overlap(Runnable first, Runnable second) throws Exception {
+ Gate gate = new Gate();
+ storage.nextWrite.set(gate);
+ FutureTask<Void> firstTask = new FutureTask<>(first, null);
+ FutureTask<Void> secondTask = new FutureTask<>(second, null);
+ Thread firstThread = Thread.ofPlatform().daemon().unstarted(firstTask);
+ Thread secondThread = Thread.ofPlatform().daemon().unstarted(secondTask);
+ firstThread.start();
+ try {
+ assertTrue(gate.entered.await(5, TimeUnit.SECONDS), "First writer did not reach storage");
+ secondThread.start();
+ ThreadStates.awaitBlockedOrDone(secondThread);
+ } finally {
+ gate.release.countDown();
+ }
+ firstTask.get(5, TimeUnit.SECONDS);
+ secondTask.get(5, TimeUnit.SECONDS);
+ }
+
+ private static final class Gate {
+ private final CountDownLatch entered = new CountDownLatch(1);
+ private final CountDownLatch release = new CountDownLatch(1);
+ }
+
+ private static final class ParkingStorage extends InMemoryStorage<String, VectorBucket> {
+ private final ObjectMapper mapper = new ObjectMapper().findAndRegisterModules();
+ private final boolean copyOnAccess;
+ private final AtomicReference<Gate> nextWrite = new AtomicReference<>();
+
+ private ParkingStorage(boolean copyOnAccess) {
+ this.copyOnAccess = copyOnAccess;
+ }
+
+ @Override
+ public Optional<VectorBucket> get(String key) {
+ return super.get(key).map(this::copy);
+ }
+
+ @Override
+ public void put(String key, VectorBucket value) {
+ Gate gate = nextWrite.getAndSet(null);
+ if (gate != null) {
+ gate.entered.countDown();
+ try {
+ assertTrue(gate.release.await(5, TimeUnit.SECONDS), "Paused write was not released");
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError("Paused write was interrupted", e);
+ }
+ }
+ super.put(key, copy(value));
+ }
+
+ private VectorBucket copy(VectorBucket value) {
+ return copyOnAccess ? mapper.convertValue(value, VectorBucket.class) : value;
+ }
+ }
+}