Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3vectors): route ListVectors instead of falling through to S3 (#2285)

Feyikemi Ogunsanya6c4d63b0a9ad
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java
index 863cdd3c6b..7731a3eb1c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsController.java
@@ -200,6 +200,24 @@ public class S3VectorsController {
Map<String, Object> metadata
) {}
+ @RegisterForReflection
+ public record ListVectorsRequest(
+ String vectorBucketName,
+ String indexName,
+ String indexArn,
+ Integer maxResults,
+ String nextToken,
+ Boolean returnData,
+ Boolean returnMetadata
+ ) {}
+
+ @RegisterForReflection
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public record ListVectorsResponse(
+ List<VectorGetResponseRepresentation> vectors,
+ String nextToken
+ ) {}
+
@RegisterForReflection
public record DeleteVectorsRequest(
String vectorBucketName,
@@ -370,6 +388,28 @@ public class S3VectorsController {
return Response.ok(new GetVectorsResponse(reps)).build();
}
+ @POST
+ @Path("/ListVectors")
+ public Response listVectors(@Context HttpHeaders headers, ListVectorsRequest request) {
+ String region = regionResolver.resolveRegion(headers);
+ int maxResults = request.maxResults() != null ? request.maxResults() : 0;
+ S3VectorsService.ListVectorsResult result = service.listVectors(
+ request.vectorBucketName(), request.indexName(), request.indexArn(), maxResults, request.nextToken(), region);
+
+ boolean returnData = request.returnData() != null && request.returnData();
+ boolean returnMetadata = request.returnMetadata() != null && request.returnMetadata();
+
+ List<VectorGetResponseRepresentation> reps = result.vectors().stream()
+ .map(v -> {
+ VectorDataRepresentation data = returnData ? new VectorDataRepresentation(v.getData()) : null;
+ Map<String, Object> metadata = returnMetadata ? v.getMetadata() : null;
+ return new VectorGetResponseRepresentation(v.getKey(), data, metadata);
+ })
+ .toList();
+
+ return Response.ok(new ListVectorsResponse(reps, result.nextToken())).build();
+ }
+
@POST
@Path("/DeleteVectors")
public Response deleteVectors(@Context HttpHeaders headers, DeleteVectorsRequest request) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java
index 5d4e8f22b2..ee39639642 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsService.java
@@ -13,6 +13,7 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@@ -154,6 +155,104 @@ public class S3VectorsService {
return result;
}
+ /**
+ * Lists the vectors stored in an index with cursor-based pagination, ordered by key.
+ * The index may be identified by {@code bucketName}/{@code indexName} or by {@code indexArn} alone,
+ * matching the ListVectors input contract.
+ */
+ public ListVectorsResult listVectors(String bucketName, String indexName, String indexArn,
+ int maxResults, String nextToken, String region) {
+ VectorIndex index = resolveIndex(bucketName, indexName, indexArn, region);
+ int limit = maxResults > 0 ? Math.min(maxResults, 1000) : 500;
+ String lastKey = decodeToken(nextToken);
+
+ List<VectorData> allVectors = index.getVectors().values().stream()
+ .sorted(Comparator.comparing(VectorData::getKey))
+ .collect(Collectors.toList());
+
+ int startIndex = 0;
+ if (lastKey != null) {
+ for (int i = 0; i < allVectors.size(); i++) {
+ if (allVectors.get(i).getKey().compareTo(lastKey) > 0) {
+ startIndex = i;
+ break;
+ }
+ if (i == allVectors.size() - 1) {
+ startIndex = allVectors.size();
+ }
+ }
+ }
+
+ List<VectorData> page = allVectors.stream()
+ .skip(startIndex)
+ .limit(limit)
+ .collect(Collectors.toList());
+
+ String newNextToken = null;
+ if (startIndex + limit < allVectors.size() && !page.isEmpty()) {
+ newNextToken = encodeToken(page.get(page.size() - 1).getKey());
+ }
+
+ return new ListVectorsResult(page, newNextToken);
+ }
+
+ /**
+ * Resolves an index by name or by ARN (format: {@code <bucketArn>/index/<indexName>}), matching
+ * ListVectors' "vectorBucketName + indexName, or indexArn alone" input contract.
+ */
+ private VectorIndex resolveIndex(String bucketName, String indexName, String indexArn, String region) {
+ if (indexName != null && !indexName.isBlank()) {
+ return getIndex(bucketName, indexName, region);
+ }
+ if (indexArn == null || indexArn.isBlank()) {
+ throw new AwsException("ValidationException", "Either indexName or indexArn is required.", 400);
+ }
+ AwsArnUtils.Arn parsed;
+ try {
+ parsed = AwsArnUtils.parse(indexArn);
+ } catch (IllegalArgumentException e) {
+ throw new AwsException("ValidationException", "Malformed indexArn: " + indexArn, 400);
+ }
+ if (!"s3vectors".equals(parsed.service())) {
+ throw new AwsException("ValidationException", "Not an S3 Vectors indexArn: " + indexArn, 400);
+ }
+ String resource = parsed.resource();
+ int marker = resource.lastIndexOf("/index/");
+ if (marker < 0 || !resource.startsWith("bucket/")) {
+ throw new AwsException("ValidationException", "Malformed indexArn: " + indexArn, 400);
+ }
+ String bucketNameFromArn = resource.substring("bucket/".length(), marker);
+ String indexNameFromArn = resource.substring(marker + "/index/".length());
+ // The ARN carries its own region — an index in one region must resolve there regardless
+ // of which region the ListVectors request itself was signed for.
+ String arnRegion = parsed.region().isEmpty() ? region : parsed.region();
+ return getIndex(bucketNameFromArn, indexNameFromArn, arnRegion);
+ }
+
+ /**
+ * Encodes a pagination cursor as an opaque Base64 token over the raw key — no JSON wrapping, so
+ * keys containing quotes or other JSON-significant characters round-trip correctly.
+ */
+ private String encodeToken(String lastKey) {
+ if (lastKey == null) {
+ return null;
+ }
+ return Base64.getEncoder().encodeToString(lastKey.getBytes(StandardCharsets.UTF_8));
+ }
+
+ private String decodeToken(String token) {
+ if (token == null || token.isEmpty()) {
+ return null;
+ }
+ try {
+ return new String(Base64.getDecoder().decode(token), StandardCharsets.UTF_8);
+ } catch (Exception e) {
+ throw new AwsException("InvalidNextTokenException", "Invalid pagination token", 400);
+ }
+ }
+
+ public record ListVectorsResult(List<VectorData> vectors, String nextToken) {}
+
public void deleteVectors(String bucketName, String indexName, List<String> keys, String region) {
VectorBucket bucket = getVectorBucket(bucketName, region);
VectorIndex index = getIndex(bucketName, indexName, region);
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsIntegrationTest.java
index bd026e6b6c..d7428dd71f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3vectors/S3VectorsIntegrationTest.java
@@ -248,6 +248,75 @@ class S3VectorsIntegrationTest {
@Test
@Order(11)
+ void listVectors() {
+ // Reproduces #2161: ListVectors previously had no route and fell through to the S3
+ // REST-XML controller, returning an S3 <Error>InvalidArgument</Error> instead of a
+ // ListVectors response.
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ {
+ "vectorBucketName": "%s",
+ "indexName": "%s",
+ "returnData": true,
+ "returnMetadata": true
+ }
+ """.formatted(BUCKET_NAME, INDEX_NAME))
+ .when()
+ .post("/ListVectors")
+ .then()
+ .contentType(JSON_CONTENT_TYPE)
+ .statusCode(200)
+ .body("vectors", hasSize(2))
+ .body("vectors.find { it.key == 'v1' }.data.float32", contains(1.0f, 0.0f, 0.0f))
+ .body("vectors.find { it.key == 'v1' }.metadata.label", equalTo("first"))
+ .body("vectors.find { it.key == 'v2' }.metadata.label", equalTo("second"))
+ .body("nextToken", nullValue());
+ }
+
+ @Test
+ @Order(12)
+ void listVectorsPaginatesWithMaxResultsAndNextToken() {
+ String firstPageNextToken =
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ {
+ "vectorBucketName": "%s",
+ "indexName": "%s",
+ "maxResults": 1
+ }
+ """.formatted(BUCKET_NAME, INDEX_NAME))
+ .when()
+ .post("/ListVectors")
+ .then()
+ .statusCode(200)
+ .body("vectors", hasSize(1))
+ .body("vectors[0].key", equalTo("v1"))
+ .body("nextToken", notNullValue())
+ .extract().path("nextToken");
+
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ {
+ "vectorBucketName": "%s",
+ "indexName": "%s",
+ "maxResults": 1,
+ "nextToken": "%s"
+ }
+ """.formatted(BUCKET_NAME, INDEX_NAME, firstPageNextToken))
+ .when()
+ .post("/ListVectors")
+ .then()
+ .statusCode(200)
+ .body("vectors", hasSize(1))
+ .body("vectors[0].key", equalTo("v2"))
+ .body("nextToken", nullValue());
+ }
+
+ @Test
+ @Order(13)
void deleteVectors() {
given()
.contentType(JSON_CONTENT_TYPE)
@@ -282,7 +351,7 @@ class S3VectorsIntegrationTest {
}
@Test
- @Order(12)
+ @Order(14)
void deleteIndex() {
given()
.contentType(JSON_CONTENT_TYPE)
@@ -313,7 +382,7 @@ class S3VectorsIntegrationTest {
}
@Test
- @Order(13)
+ @Order(15)
void deleteVectorBucket() {
given()
.contentType(JSON_CONTENT_TYPE)
@@ -340,4 +409,229 @@ class S3VectorsIntegrationTest {
.then()
.statusCode(404);
}
+
+ // Self-contained: each test below creates its own bucket/index so it doesn't depend on the
+ // ordered setup/teardown sequence above.
+
+ @Test
+ @Order(16)
+ void listVectorsByIndexArnOnly() {
+ String bucketName = "arn-list-vectors-bucket";
+ String indexName = "arn-idx";
+
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ { "vectorBucketName": "%s" }
+ """.formatted(bucketName))
+ .when()
+ .post("/CreateVectorBucket")
+ .then()
+ .statusCode(200);
+
+ String indexArn =
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ {
+ "vectorBucketName": "%s",
+ "indexName": "%s",
+ "dimension": 2,
+ "distanceMetric": "cosine",
+ "dataType": "float32"
+ }
+ """.formatted(bucketName, indexName))
+ .when()
+ .post("/CreateIndex")
+ .then()
+ .statusCode(200)
+ .extract().path("indexArn");
+
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ {
+ "vectorBucketName": "%s",
+ "indexName": "%s",
+ "vectors": [{"key": "only", "data": {"float32": [1.0, 0.0]}}]
+ }
+ """.formatted(bucketName, indexName))
+ .when()
+ .post("/PutVectors")
+ .then()
+ .statusCode(200);
+
+ // ListVectors identified by indexArn alone (no vectorBucketName/indexName) must resolve
+ // to the same index rather than looking up a null bucket name.
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ { "indexArn": "%s" }
+ """.formatted(indexArn))
+ .when()
+ .post("/ListVectors")
+ .then()
+ .statusCode(200)
+ .body("vectors", hasSize(1))
+ .body("vectors[0].key", equalTo("only"));
+ }
+
+ @Test
+ @Order(17)
+ void listVectorsPaginationRoundTripsKeysWithQuotes() {
+ String bucketName = "quote-key-vector-bucket";
+ String indexName = "quote-idx";
+
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ { "vectorBucketName": "%s" }
+ """.formatted(bucketName))
+ .when()
+ .post("/CreateVectorBucket")
+ .then()
+ .statusCode(200);
+
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ {
+ "vectorBucketName": "%s",
+ "indexName": "%s",
+ "dimension": 2,
+ "distanceMetric": "cosine",
+ "dataType": "float32"
+ }
+ """.formatted(bucketName, indexName))
+ .when()
+ .post("/CreateIndex")
+ .then()
+ .statusCode(200);
+
+ // One key contains a double quote — the pagination cursor must round-trip it intact
+ // rather than truncating at the quote.
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ {
+ "vectorBucketName": "%s",
+ "indexName": "%s",
+ "vectors": [
+ {"key": "a\\"b", "data": {"float32": [1.0, 0.0]}},
+ {"key": "c", "data": {"float32": [0.0, 1.0]}}
+ ]
+ }
+ """.formatted(bucketName, indexName))
+ .when()
+ .post("/PutVectors")
+ .then()
+ .statusCode(200);
+
+ String firstPageNextToken =
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ {
+ "vectorBucketName": "%s",
+ "indexName": "%s",
+ "maxResults": 1
+ }
+ """.formatted(bucketName, indexName))
+ .when()
+ .post("/ListVectors")
+ .then()
+ .statusCode(200)
+ .body("vectors", hasSize(1))
+ .body("vectors[0].key", equalTo("a\"b"))
+ .body("nextToken", notNullValue())
+ .extract().path("nextToken");
+
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ {
+ "vectorBucketName": "%s",
+ "indexName": "%s",
+ "maxResults": 1,
+ "nextToken": "%s"
+ }
+ """.formatted(bucketName, indexName, firstPageNextToken))
+ .when()
+ .post("/ListVectors")
+ .then()
+ .statusCode(200)
+ .body("vectors", hasSize(1))
+ .body("vectors[0].key", equalTo("c"))
+ .body("nextToken", nullValue());
+ }
+
+ @Test
+ @Order(18)
+ void listVectorsByIndexArnResolvesTheArnsOwnRegionNotTheRequestRegion() {
+ String bucketName = "cross-region-vector-bucket";
+ String indexName = "cross-region-idx";
+ String otherRegionAuth = "Credential=AKID/20260101/us-west-2/s3vectors/aws4_request";
+
+ // Create the bucket/index in us-west-2 (via the SigV4 credential scope region).
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .header("Authorization", otherRegionAuth)
+ .body("""
+ { "vectorBucketName": "%s" }
+ """.formatted(bucketName))
+ .when()
+ .post("/CreateVectorBucket")
+ .then()
+ .statusCode(200);
+
+ String indexArn =
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .header("Authorization", otherRegionAuth)
+ .body("""
+ {
+ "vectorBucketName": "%s",
+ "indexName": "%s",
+ "dimension": 2,
+ "distanceMetric": "cosine",
+ "dataType": "float32"
+ }
+ """.formatted(bucketName, indexName))
+ .when()
+ .post("/CreateIndex")
+ .then()
+ .statusCode(200)
+ .body("indexArn", containsString(":us-west-2:"))
+ .extract().path("indexArn");
+
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .header("Authorization", otherRegionAuth)
+ .body("""
+ {
+ "vectorBucketName": "%s",
+ "indexName": "%s",
+ "vectors": [{"key": "only", "data": {"float32": [1.0, 0.0]}}]
+ }
+ """.formatted(bucketName, indexName))
+ .when()
+ .post("/PutVectors")
+ .then()
+ .statusCode(200);
+
+ // No Authorization header here, so this request resolves to the default region — not
+ // us-west-2. The lookup must still succeed because it honors the region encoded in the
+ // ARN itself, not the region the ListVectors request happens to be signed for.
+ given()
+ .contentType(JSON_CONTENT_TYPE)
+ .body("""
+ { "indexArn": "%s" }
+ """.formatted(indexArn))
+ .when()
+ .post("/ListVectors")
+ .then()
+ .statusCode(200)
+ .body("vectors", hasSize(1))
+ .body("vectors[0].key", equalTo("only"));
+ }
}