Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3vectors): route ListVectors instead of falling through to S3 (#2285)
Feyikemi Ogunsanya6c4d63b0a9ad
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.javaindex 863cdd3c6b..7731a3eb1c 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java@@ -200,6 +200,24 @@ public class S3VectorsController {Map<String, Object> metadata) {}+ @RegisterForReflection+ public record ListVectorsRequest(+ String vectorBucketName,+ String indexName,+ String indexArn,+ Integer maxResults,+ String nextToken,+ Boolean returnData,+ Boolean returnMetadata+ ) {}++ @RegisterForReflection+ @JsonInclude(JsonInclude.Include.NON_NULL)+ public record ListVectorsResponse(+ List<VectorGetResponseRepresentation> vectors,+ String nextToken+ ) {}+@RegisterForReflectionpublic record DeleteVectorsRequest(String vectorBucketName,@@ -370,6 +388,28 @@ public class S3VectorsController {return Response.ok(new GetVectorsResponse(reps)).build();}+ @POST+ @Path("/ListVectors")+ public Response listVectors(@Context HttpHeaders headers, ListVectorsRequest request) {+ String region = regionResolver.resolveRegion(headers);+ int maxResults = request.maxResults() != null ? request.maxResults() : 0;+ S3VectorsService.ListVectorsResult result = service.listVectors(+ request.vectorBucketName(), request.indexName(), request.indexArn(), maxResults, request.nextToken(), region);++ boolean returnData = request.returnData() != null && request.returnData();+ boolean returnMetadata = request.returnMetadata() != null && request.returnMetadata();++ List<VectorGetResponseRepresentation> reps = result.vectors().stream()+ .map(v -> {+ VectorDataRepresentation data = returnData ? new VectorDataRepresentation(v.getData()) : null;+ Map<String, Object> metadata = returnMetadata ? v.getMetadata() : null;+ return new VectorGetResponseRepresentation(v.getKey(), data, metadata);+ })+ .toList();++ return Response.ok(new ListVectorsResponse(reps, result.nextToken())).build();+ }+@POST@Path("/DeleteVectors")public Response deleteVectors(@Context HttpHeaders headers, DeleteVectorsRequest request) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.javaindex 5d4e8f22b2..ee39639642 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java@@ -13,6 +13,7 @@ import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.nio.charset.StandardCharsets;import java.util.*;import java.util.concurrent.ConcurrentHashMap;import java.util.stream.Collectors;@@ -154,6 +155,104 @@ public class S3VectorsService {return result;}+ /**+ * Lists the vectors stored in an index with cursor-based pagination, ordered by key.+ * The index may be identified by {@code bucketName}/{@code indexName} or by {@code indexArn} alone,+ * matching the ListVectors input contract.+ */+ public ListVectorsResult listVectors(String bucketName, String indexName, String indexArn,+ int maxResults, String nextToken, String region) {+ VectorIndex index = resolveIndex(bucketName, indexName, indexArn, region);+ int limit = maxResults > 0 ? Math.min(maxResults, 1000) : 500;+ String lastKey = decodeToken(nextToken);++ List<VectorData> allVectors = index.getVectors().values().stream()+ .sorted(Comparator.comparing(VectorData::getKey))+ .collect(Collectors.toList());++ int startIndex = 0;+ if (lastKey != null) {+ for (int i = 0; i < allVectors.size(); i++) {+ if (allVectors.get(i).getKey().compareTo(lastKey) > 0) {+ startIndex = i;+ break;+ }+ if (i == allVectors.size() - 1) {+ startIndex = allVectors.size();+ }+ }+ }++ List<VectorData> page = allVectors.stream()+ .skip(startIndex)+ .limit(limit)+ .collect(Collectors.toList());++ String newNextToken = null;+ if (startIndex + limit < allVectors.size() && !page.isEmpty()) {+ newNextToken = encodeToken(page.get(page.size() - 1).getKey());+ }++ return new ListVectorsResult(page, newNextToken);+ }++ /**+ * Resolves an index by name or by ARN (format: {@code <bucketArn>/index/<indexName>}), matching+ * ListVectors' "vectorBucketName + indexName, or indexArn alone" input contract.+ */+ private VectorIndex resolveIndex(String bucketName, String indexName, String indexArn, String region) {+ if (indexName != null && !indexName.isBlank()) {+ return getIndex(bucketName, indexName, region);+ }+ if (indexArn == null || indexArn.isBlank()) {+ throw new AwsException("ValidationException", "Either indexName or indexArn is required.", 400);+ }+ AwsArnUtils.Arn parsed;+ try {+ parsed = AwsArnUtils.parse(indexArn);+ } catch (IllegalArgumentException e) {+ throw new AwsException("ValidationException", "Malformed indexArn: " + indexArn, 400);+ }+ if (!"s3vectors".equals(parsed.service())) {+ throw new AwsException("ValidationException", "Not an S3 Vectors indexArn: " + indexArn, 400);+ }+ String resource = parsed.resource();+ int marker = resource.lastIndexOf("/index/");+ if (marker < 0 || !resource.startsWith("bucket/")) {+ throw new AwsException("ValidationException", "Malformed indexArn: " + indexArn, 400);+ }+ String bucketNameFromArn = resource.substring("bucket/".length(), marker);+ String indexNameFromArn = resource.substring(marker + "/index/".length());+ // The ARN carries its own region — an index in one region must resolve there regardless+ // of which region the ListVectors request itself was signed for.+ String arnRegion = parsed.region().isEmpty() ? region : parsed.region();+ return getIndex(bucketNameFromArn, indexNameFromArn, arnRegion);+ }++ /**+ * Encodes a pagination cursor as an opaque Base64 token over the raw key — no JSON wrapping, so+ * keys containing quotes or other JSON-significant characters round-trip correctly.+ */+ private String encodeToken(String lastKey) {+ if (lastKey == null) {+ return null;+ }+ return Base64.getEncoder().encodeToString(lastKey.getBytes(StandardCharsets.UTF_8));+ }++ private String decodeToken(String token) {+ if (token == null || token.isEmpty()) {+ return null;+ }+ try {+ return new String(Base64.getDecoder().decode(token), StandardCharsets.UTF_8);+ } catch (Exception e) {+ throw new AwsException("InvalidNextTokenException", "Invalid pagination token", 400);+ }+ }++ public record ListVectorsResult(List<VectorData> vectors, String nextToken) {}+public void deleteVectors(String bucketName, String indexName, List<String> keys, String region) {VectorBucket bucket = getVectorBucket(bucketName, region);VectorIndex index = getIndex(bucketName, indexName, region);diff --git a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsIntegrationTest.javaindex bd026e6b6c..d7428dd71f 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsIntegrationTest.java@@ -248,6 +248,75 @@ class S3VectorsIntegrationTest {@Test@Order(11)+ void listVectors() {+ // Reproduces #2161: ListVectors previously had no route and fell through to the S3+ // REST-XML controller, returning an S3 <Error>InvalidArgument</Error> instead of a+ // ListVectors response.+ given()+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ {+ "vectorBucketName": "%s",+ "indexName": "%s",+ "returnData": true,+ "returnMetadata": true+ }+ """.formatted(BUCKET_NAME, INDEX_NAME))+ .when()+ .post("/ListVectors")+ .then()+ .contentType(JSON_CONTENT_TYPE)+ .statusCode(200)+ .body("vectors", hasSize(2))+ .body("vectors.find { it.key == 'v1' }.data.float32", contains(1.0f, 0.0f, 0.0f))+ .body("vectors.find { it.key == 'v1' }.metadata.label", equalTo("first"))+ .body("vectors.find { it.key == 'v2' }.metadata.label", equalTo("second"))+ .body("nextToken", nullValue());+ }++ @Test+ @Order(12)+ void listVectorsPaginatesWithMaxResultsAndNextToken() {+ String firstPageNextToken =+ given()+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ {+ "vectorBucketName": "%s",+ "indexName": "%s",+ "maxResults": 1+ }+ """.formatted(BUCKET_NAME, INDEX_NAME))+ .when()+ .post("/ListVectors")+ .then()+ .statusCode(200)+ .body("vectors", hasSize(1))+ .body("vectors[0].key", equalTo("v1"))+ .body("nextToken", notNullValue())+ .extract().path("nextToken");++ given()+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ {+ "vectorBucketName": "%s",+ "indexName": "%s",+ "maxResults": 1,+ "nextToken": "%s"+ }+ """.formatted(BUCKET_NAME, INDEX_NAME, firstPageNextToken))+ .when()+ .post("/ListVectors")+ .then()+ .statusCode(200)+ .body("vectors", hasSize(1))+ .body("vectors[0].key", equalTo("v2"))+ .body("nextToken", nullValue());+ }++ @Test+ @Order(13)void deleteVectors() {given().contentType(JSON_CONTENT_TYPE)@@ -282,7 +351,7 @@ class S3VectorsIntegrationTest {}@Test- @Order(12)+ @Order(14)void deleteIndex() {given().contentType(JSON_CONTENT_TYPE)@@ -313,7 +382,7 @@ class S3VectorsIntegrationTest {}@Test- @Order(13)+ @Order(15)void deleteVectorBucket() {given().contentType(JSON_CONTENT_TYPE)@@ -340,4 +409,229 @@ class S3VectorsIntegrationTest {.then().statusCode(404);}++ // Self-contained: each test below creates its own bucket/index so it doesn't depend on the+ // ordered setup/teardown sequence above.++ @Test+ @Order(16)+ void listVectorsByIndexArnOnly() {+ String bucketName = "arn-list-vectors-bucket";+ String indexName = "arn-idx";++ given()+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ { "vectorBucketName": "%s" }+ """.formatted(bucketName))+ .when()+ .post("/CreateVectorBucket")+ .then()+ .statusCode(200);++ String indexArn =+ given()+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ {+ "vectorBucketName": "%s",+ "indexName": "%s",+ "dimension": 2,+ "distanceMetric": "cosine",+ "dataType": "float32"+ }+ """.formatted(bucketName, indexName))+ .when()+ .post("/CreateIndex")+ .then()+ .statusCode(200)+ .extract().path("indexArn");++ given()+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ {+ "vectorBucketName": "%s",+ "indexName": "%s",+ "vectors": [{"key": "only", "data": {"float32": [1.0, 0.0]}}]+ }+ """.formatted(bucketName, indexName))+ .when()+ .post("/PutVectors")+ .then()+ .statusCode(200);++ // ListVectors identified by indexArn alone (no vectorBucketName/indexName) must resolve+ // to the same index rather than looking up a null bucket name.+ given()+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ { "indexArn": "%s" }+ """.formatted(indexArn))+ .when()+ .post("/ListVectors")+ .then()+ .statusCode(200)+ .body("vectors", hasSize(1))+ .body("vectors[0].key", equalTo("only"));+ }++ @Test+ @Order(17)+ void listVectorsPaginationRoundTripsKeysWithQuotes() {+ String bucketName = "quote-key-vector-bucket";+ String indexName = "quote-idx";++ given()+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ { "vectorBucketName": "%s" }+ """.formatted(bucketName))+ .when()+ .post("/CreateVectorBucket")+ .then()+ .statusCode(200);++ given()+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ {+ "vectorBucketName": "%s",+ "indexName": "%s",+ "dimension": 2,+ "distanceMetric": "cosine",+ "dataType": "float32"+ }+ """.formatted(bucketName, indexName))+ .when()+ .post("/CreateIndex")+ .then()+ .statusCode(200);++ // One key contains a double quote — the pagination cursor must round-trip it intact+ // rather than truncating at the quote.+ given()+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ {+ "vectorBucketName": "%s",+ "indexName": "%s",+ "vectors": [+ {"key": "a\\"b", "data": {"float32": [1.0, 0.0]}},+ {"key": "c", "data": {"float32": [0.0, 1.0]}}+ ]+ }+ """.formatted(bucketName, indexName))+ .when()+ .post("/PutVectors")+ .then()+ .statusCode(200);++ String firstPageNextToken =+ given()+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ {+ "vectorBucketName": "%s",+ "indexName": "%s",+ "maxResults": 1+ }+ """.formatted(bucketName, indexName))+ .when()+ .post("/ListVectors")+ .then()+ .statusCode(200)+ .body("vectors", hasSize(1))+ .body("vectors[0].key", equalTo("a\"b"))+ .body("nextToken", notNullValue())+ .extract().path("nextToken");++ given()+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ {+ "vectorBucketName": "%s",+ "indexName": "%s",+ "maxResults": 1,+ "nextToken": "%s"+ }+ """.formatted(bucketName, indexName, firstPageNextToken))+ .when()+ .post("/ListVectors")+ .then()+ .statusCode(200)+ .body("vectors", hasSize(1))+ .body("vectors[0].key", equalTo("c"))+ .body("nextToken", nullValue());+ }++ @Test+ @Order(18)+ void listVectorsByIndexArnResolvesTheArnsOwnRegionNotTheRequestRegion() {+ String bucketName = "cross-region-vector-bucket";+ String indexName = "cross-region-idx";+ String otherRegionAuth = "Credential=AKID/20260101/us-west-2/s3vectors/aws4_request";++ // Create the bucket/index in us-west-2 (via the SigV4 credential scope region).+ given()+ .contentType(JSON_CONTENT_TYPE)+ .header("Authorization", otherRegionAuth)+ .body("""+ { "vectorBucketName": "%s" }+ """.formatted(bucketName))+ .when()+ .post("/CreateVectorBucket")+ .then()+ .statusCode(200);++ String indexArn =+ given()+ .contentType(JSON_CONTENT_TYPE)+ .header("Authorization", otherRegionAuth)+ .body("""+ {+ "vectorBucketName": "%s",+ "indexName": "%s",+ "dimension": 2,+ "distanceMetric": "cosine",+ "dataType": "float32"+ }+ """.formatted(bucketName, indexName))+ .when()+ .post("/CreateIndex")+ .then()+ .statusCode(200)+ .body("indexArn", containsString(":us-west-2:"))+ .extract().path("indexArn");++ given()+ .contentType(JSON_CONTENT_TYPE)+ .header("Authorization", otherRegionAuth)+ .body("""+ {+ "vectorBucketName": "%s",+ "indexName": "%s",+ "vectors": [{"key": "only", "data": {"float32": [1.0, 0.0]}}]+ }+ """.formatted(bucketName, indexName))+ .when()+ .post("/PutVectors")+ .then()+ .statusCode(200);++ // No Authorization header here, so this request resolves to the default region — not+ // us-west-2. The lookup must still succeed because it honors the region encoded in the+ // ARN itself, not the region the ListVectors request happens to be signed for.+ given()+ .contentType(JSON_CONTENT_TYPE)+ .body("""+ { "indexArn": "%s" }+ """.formatted(indexArn))+ .when()+ .post("/ListVectors")+ .then()+ .statusCode(200)+ .body("vectors", hasSize(1))+ .body("vectors[0].key", equalTo("only"));+ }}